[PLAT-16521]Create swagger_all.json that includes INTERNAL APIs to be consumed in CLI

Summary: Creating swagger-all.json that contains all APIs (public, preview, deprecated, internal) that can be consumed by CLI and Terraform.

Test Plan:
Check swagger-all.json file.
Check the `client/go/` directory for all the APIs missing due to the YbaApi.Internal tag

Reviewers: #yba-api-review, sneelakantan, skurapati

Reviewed By: #yba-api-review, sneelakantan

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D41370

Author: Deepti-yb

managed/.gitignore

@@ -71,6 +71,7 @@ devops/pex/pexEnv
 # openapi generated server stubs during build
 src/main/resources/openapi.yaml
 src/main/resources/openapi_public.yaml
+src/main/resources/swagger-all.json
 tmp
 
 # openapi generated JS API stubs

managed/build.sbt

@@ -686,7 +686,7 @@ lazy val pythonGenV2Client = project.in(file("client/python"))
 // Generate a Go API client.
 lazy val gogen = project.in(file("client/go"))
   .settings(
-    openApiInputSpec := "src/main/resources/swagger.json",
+    openApiInputSpec := "src/main/resources/swagger-all.json",
     openApiGeneratorName := "go",
     openApiOutputDir := "client/go/v1",
     openApiGenerateModelTests := SettingDisabled,
@@ -1146,6 +1146,7 @@ lazy val swagger = project
       // Consider generating this only in managedResources
       val swaggerJson = (root / Compile / resourceDirectory).value / "swagger.json"
       val swaggerStrictJson = (root / Compile / resourceDirectory).value / "swagger-strict.json"
+      val swaggerAllJson = (root / Compile / resourceDirectory).value / "swagger-all.json"
       Def.sequential(
         (Test / runMain )
           .toTask(s" com.yugabyte.yw.controllers.SwaggerGenTest $swaggerJson"),
@@ -1156,6 +1157,8 @@ lazy val swagger = project
         // or use '--exclude_deprecated all' to drop all deprecated APIs
         (Test / runMain )
           .toTask(s" com.yugabyte.yw.controllers.SwaggerGenTest $swaggerStrictJson --exclude_deprecated all"),
+        (Test / runMain )
+          .toTask(s" com.yugabyte.yw.controllers.SwaggerGenTest $swaggerAllJson --exclude_internal none")
       )
     }.value,
 

managed/swagger/src/test/java/com/yugabyte/yw/common/swagger/PlatformModelConverter.java

@@ -31,11 +31,15 @@ public class PlatformModelConverter implements ModelConverter {
   public static String excludeYbaDeprecatedOption;
   private static YBADeprecationProcessor deprecationProcessor;
 
+  public static String excludeYbaInternalOption;
+  private static YBAInternalProcessor internalProcessor;
+
   static void register() {
     // remove if one is already registers to avoid duplicates in tests
     ModelConverters.getInstance().removeConverter(SINGLETON);
     ModelConverters.getInstance().addConverter(SINGLETON);
     deprecationProcessor = new YBADeprecationProcessor(excludeYbaDeprecatedOption);
+    internalProcessor = new YBAInternalProcessor(excludeYbaInternalOption);
   }
 
   private static final ImmutableSet<String> SKIPPED_PACKAGES =

managed/swagger/src/test/java/com/yugabyte/yw/common/swagger/PlatformSwaggerSpecFilter.java

@@ -18,10 +18,12 @@ public class PlatformSwaggerSpecFilter extends AbstractSpecFilter {
   private static final String INTERNAL_MARKER = "YbaApi Internal";
   private static final String DEPRECATED_MARKER = "Deprecated since YBA version";
   private boolean isStrictMode = false;
+  private boolean isInternalMode = false;
 
   public PlatformSwaggerSpecFilter() {
     PlatformModelConverter.register();
     isStrictMode = "all".equalsIgnoreCase(PlatformModelConverter.excludeYbaDeprecatedOption);
+    isInternalMode = "none".equalsIgnoreCase(PlatformModelConverter.excludeYbaInternalOption);
   }
 
   @Override
@@ -31,14 +33,17 @@ public boolean isOperationAllowed(
       Map<String, List<String>> params,
       Map<String, String> cookies,
       Map<String, List<String>> headers) {
-    if (operation.getDescription() != null
-        && operation.getDescription().contains(INTERNAL_MARKER)) {
-      LOG.info("Skipping swagger generation for internal method '{}'", operation.getOperationId());
-      return false;
+    if (!isInternalMode) {
+      if (operation.getDescription() != null
+          && operation.getDescription().contains(INTERNAL_MARKER)) {
+        LOG.info(
+            "Skipping swagger generation for internal method '{}'", operation.getOperationId());
+        return false;
+      }
     }
     if (isStrictMode) {
-        if (operation.getDescription() != null
-            && operation.getDescription().contains(DEPRECATED_MARKER)) {
+      if (operation.getDescription() != null
+          && operation.getDescription().contains(DEPRECATED_MARKER)) {
         LOG.info("Skipping deprecated method in strict mode '{}'", operation.getOperationId());
         return false;
       }
@@ -54,13 +59,15 @@ public boolean isParamAllowed(
       Map<String, List<String>> params,
       Map<String, String> cookies,
       Map<String, List<String>> headers) {
-    if (parameter.getDescription() != null
-        && parameter.getDescription().contains(INTERNAL_MARKER)) {
-      LOG.info(
-          "Skipping swagger generation for internal param '{}' of operation '{}'",
-          parameter.getName(),
-          operation.getOperationId());
-      return false;
+    if (!isInternalMode) {
+      if (parameter.getDescription() != null
+          && parameter.getDescription().contains(INTERNAL_MARKER)) {
+        LOG.info(
+            "Skipping swagger generation for internal param '{}' of operation '{}'",
+            parameter.getName(),
+            operation.getOperationId());
+        return false;
+      }
     }
     if (isStrictMode) {
       if (parameter.getDescription() != null
@@ -81,9 +88,11 @@ public boolean isDefinitionAllowed(
       Map<String, List<String>> params,
       Map<String, String> cookies,
       Map<String, List<String>> headers) {
-    if (model.getDescription() != null && model.getDescription().contains(INTERNAL_MARKER)) {
-      LOG.info("Skipping swagger generation for model '{}'", ((ModelImpl) model).getName());
-      return false;
+    if (!isInternalMode) {
+      if (model.getDescription() != null && model.getDescription().contains(INTERNAL_MARKER)) {
+        LOG.info("Skipping swagger generation for model '{}'", ((ModelImpl) model).getName());
+        return false;
+      }
     }
     if (isStrictMode) {
       if (model.getDescription() != null && model.getDescription().contains(DEPRECATED_MARKER)) {
@@ -102,14 +111,18 @@ public boolean isPropertyAllowed(
       Map<String, List<String>> params,
       Map<String, String> cookies,
       Map<String, List<String>> headers) {
-    if (property.getDescription() != null && property.getDescription().contains(INTERNAL_MARKER)) {
-      LOG.info(
-          "Skipping swagger generation for property '{}' of model '{}'",
-          property.getName(),
-          ((ModelImpl) model).getName());
-      return false;
+    if (!isInternalMode) {
+      if (property.getDescription() != null
+          && property.getDescription().contains(INTERNAL_MARKER)) {
+        LOG.info(
+            "Skipping swagger generation for property '{}' of model '{}'",
+            property.getName(),
+            ((ModelImpl) model).getName());
+        return false;
+      }
     }
-    // TODO: move deprecation skipping of model property from PlatformModelConverter to here
+    // TODO: move deprecation skipping of model property from PlatformModelConverter
+    // to here
     return true;
   }
 }

managed/swagger/src/test/java/com/yugabyte/yw/common/swagger/YBAInternalProcessor.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2023 YugaByte, Inc. and Contributors
+ *
+ * Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt
+ */
+package com.yugabyte.yw.common.swagger;
+
+import com.google.common.base.Strings;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class YBAInternalProcessor {
+  private String excludeYbaInternal;
+
+  enum ExcludeKind {
+    NONE,
+    ALL,
+  }
+
+  private ExcludeKind excludeKind = ExcludeKind.NONE;
+
+  /*
+   * @param excludeExpression can be set to something like - "all", "none"
+   */
+  public YBAInternalProcessor(String excludeExpression) {
+    setExcludeInternal(excludeExpression);
+  }
+
+  private void setExcludeInternal(String excludeExpression) {
+    if (Strings.isNullOrEmpty(excludeExpression)
+        || excludeExpression.trim().equalsIgnoreCase("all")) {
+      excludeKind = ExcludeKind.ALL;
+      log.info("Setting Internal to exclude ALL");
+      return;
+    }
+    if (excludeExpression.trim().equalsIgnoreCase("none")) {
+      excludeKind = ExcludeKind.NONE;
+      log.info("Setting Internal to exclude NONE");
+      return;
+    }
+    throwInvalidExcludeExpression(excludeExpression);
+  }
+
+  private void throwInvalidExcludeExpression(String excludeExpression) {
+    String errMsg = String.format("--exclude-internal=%s is invalid", excludeExpression);
+    throw new RuntimeException(errMsg);
+  }
+
+  public boolean shouldExcludeInternal() {
+    switch (excludeKind) {
+      case ALL:
+        return true;
+      case NONE:
+        return false;
+      default:
+        return true; // exclude all internal APIs by default
+    }
+  }
+
+  // For unit testing
+  ExcludeKind getExcludeKind() {
+    return excludeKind;
+  }
+}

managed/swagger/src/test/java/com/yugabyte/yw/controllers/SwaggerGenTest.java

@@ -346,8 +346,16 @@ private void sortTagsList(ObjectNode specJsonNode) {
 
   public static void main(String[] args)
       throws IOException, NoSuchFieldException, IllegalAccessException {
-    String expectedSwagger = getCurrentSpec(args[0]);
     excludeDeprecated(args);
+    excludeInternal(args);
+    // Not checking swagger has changed for swagger-all.json
+    Boolean checkSwaggerHasChanged = true;
+    if (PlatformModelConverter.excludeYbaInternalOption != null) {
+      if (PlatformModelConverter.excludeYbaInternalOption.length() != 0
+          && PlatformModelConverter.excludeYbaInternalOption.toLowerCase().equals("none")) {
+        checkSwaggerHasChanged = false;
+      }
+    }
     SwaggerGenTest swaggerGenTest = new SwaggerGenTest();
     try {
       swaggerGenTest.startServer();
@@ -364,10 +372,13 @@ public static void main(String[] args)
       }
       System.out.println("Generating swagger spec:" + Arrays.toString(args));
       String swaggerSpec = swaggerGenTest.getSwaggerSpec();
-      if (expectedSwagger.length() == swaggerSpec.length()) {
-        // TODO: Fix this: Only length comparison because the json ordering change
-        System.out.println("Swagger Specs have not changed");
-        return;
+      if (checkSwaggerHasChanged) {
+        String expectedSwagger = getCurrentSpec(args[0]);
+        if (expectedSwagger.length() == swaggerSpec.length()) {
+          // TODO: Fix this: Only length comparison because the json ordering change
+          System.out.println("Swagger Specs have not changed");
+          return;
+        }
       }
       try (FileWriter fileWriter = new FileWriter(new File(args[0]))) {
         // todo only generate on change
@@ -393,6 +404,17 @@ private static void excludeDeprecated(String[] args) {
     PlatformModelConverter.excludeYbaDeprecatedOption = "";
     if (args.length > 2 && args[1].equalsIgnoreCase("--exclude_deprecated")) {
       PlatformModelConverter.excludeYbaDeprecatedOption = args[2];
+    } else if (args.length > 4 && args[3].equalsIgnoreCase("--exclude_deprecated")) {
+      PlatformModelConverter.excludeYbaDeprecatedOption = args[4];
+    }
+  }
+
+  private static void excludeInternal(String[] args) {
+    PlatformModelConverter.excludeYbaInternalOption = "";
+    if (args.length > 2 && args[1].equalsIgnoreCase("--exclude_internal")) {
+      PlatformModelConverter.excludeYbaInternalOption = args[2];
+    } else if (args.length > 4 && args[3].equalsIgnoreCase("--exclude_internal")) {
+      PlatformModelConverter.excludeYbaInternalOption = args[4];
     }
   }
 }