Fix mgmt documentation generation (Azure#30046)

* dynamically discover multi-apis instead of relying on static list
* add sphinx keyword to known words list

Author: scbedd

.vscode/cspell.json

@@ -401,6 +401,12 @@
     "aiter"
   ],
   "overrides": [
+    {
+      "filename": "doc/sphinx/generate_doc.py",
+      "words": [
+        "undoc"
+      ]
+    },
     {
       "filename": "doc/dev/test_proxy_migration_guide.md",
       "words": [

doc/sphinx/generate_doc.py

@@ -3,15 +3,21 @@
 import json
 from pathlib import Path
 import os
+import glob
 
-CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), './package_service_mapping.json')
-GENERATED_PACKAGES_LIST_FILE = 'toc_tree.rst'
+import typing
+from typing import Dict, List
+
+CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "package_service_mapping.json")
+GENERATED_PACKAGES_LIST_FILE = "toc_tree.rst"
 
 _LOGGER = logging.getLogger(__name__)
 
+
 def make_title(title):
     """Create a underlined title with the correct number of =."""
-    return "\n".join([title, len(title)*"="])
+    return "\n".join([title, len(title) * "="])
+
 
 SUBMODULE_TEMPLATE = """{title}
 
@@ -85,14 +91,42 @@ def make_title(title):
     "azure.mgmt.resource.subscriptions",
 ]
 
-def generate_doc(config_path, output_directory = "./ref/", project_pattern=None):
-    multiapi_found_apiversion = {}
-
-    rst_path_template = os.path.join(output_directory, '{}.rst')
-    rst_namespace_template = os.path.join(output_directory, '{}.{}.rst')
 
+def get_updated_config(config_path: str, package_root: str) -> Dict[str, Dict[str, typing.Union[str, List[str]]]]:
     with Path(config_path).open() as config_fd:
         config = json.load(config_fd)
+
+    namespace_folders = os.path.basename(package_root).split("-")
+    package_name = "-".join(namespace_folders)
+    namespace = ".".join(namespace_folders)
+
+    api_directory = os.path.join(package_root, *namespace_folders)
+    glob_path = os.path.join(api_directory, "v20*/")
+    valid_versions = glob.glob(glob_path)
+
+    for version_path in valid_versions:
+        api_version = os.path.relpath(version_path, start=api_directory)
+        full_namespace = namespace + f".{api_version}"
+
+        if package_name in config:
+            if "namespaces" in config[package_name]:
+                ns = config[package_name]["namespaces"]
+                if ns:
+                    if full_namespace not in ns:
+                        ns.append(full_namespace)
+
+    return config
+
+
+def generate_doc(config_path: str, output_directory: str = "./ref/", package_root: str = None) -> None:
+    multiapi_found_apiversion = {}
+    # we are handed a directory that looks like <path-to-repo>/sdk/containerservice/azure-mgmt-containerservice/
+    project_pattern = os.path.basename(package_root).replace("-", ".")
+    rst_path_template = os.path.join(output_directory, "{}.rst")
+    rst_namespace_template = os.path.join(output_directory, "{}.{}.rst")
+
+    config = get_updated_config(config_path, package_root)
+
     package_list_path = []
 
     namespaces = [n for pack in config.values() for n in pack.get("namespaces", {})]
@@ -105,25 +139,22 @@ def generate_doc(config_path, output_directory = "./ref/", project_pattern=None)
         _LOGGER.info("Working on %s", namespace)
 
         rst_path = rst_path_template.format(namespace)
-        with Path(rst_path).open('w') as rst_file:
-            rst_file.write(PACKAGE_TEMPLATE.format(
-                title=make_title(namespace+" package"),
-                namespace=namespace
-            ))
+        with Path(rst_path).open("w") as rst_file:
+            rst_file.write(PACKAGE_TEMPLATE.format(title=make_title(namespace + " package"), namespace=namespace))
 
         for module in ["operations", "models"]:
-            with Path(rst_namespace_template.format(namespace, module)).open('w') as rst_file:
-                rst_file.write(SUBMODULE_TEMPLATE.format(
-                    title=make_title(namespace+"."+module+" module"),
-                    namespace=namespace,
-                    submodule=module
-                ))
+            with Path(rst_namespace_template.format(namespace, module)).open("w") as rst_file:
+                rst_file.write(
+                    SUBMODULE_TEMPLATE.format(
+                        title=make_title(namespace + "." + module + " module"), namespace=namespace, submodule=module
+                    )
+                )
 
         for multiapi_namespace in MULTIAPI_VERSION_NAMESPACE:
             length = len(multiapi_namespace.split("."))
             if namespace.split(".")[0:length] == multiapi_namespace.split(".")[0:length]:
                 _LOGGER.info("MultiAPI namespace on %s", multiapi_namespace)
-                api_package = namespace.split(multiapi_namespace+".")[1]
+                api_package = namespace.split(multiapi_namespace + ".")[1]
                 multiapi_found_apiversion.setdefault(multiapi_namespace, []).append(api_package)
                 break
         else:
@@ -133,47 +164,47 @@ def generate_doc(config_path, output_directory = "./ref/", project_pattern=None)
         apilist.sort()
         apilist.reverse()
         rst_path = rst_path_template.format(multiapi_namespace)
-        with Path(rst_path).open('w') as rst_file:
-            rst_file.write(MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
-                title=make_title(multiapi_namespace+" package"),
-                namespace=multiapi_namespace
-            ))
+        with Path(rst_path).open("w") as rst_file:
+            rst_file.write(
+                MULTIAPI_VERSION_PACKAGE_TEMPLATE.format(
+                    title=make_title(multiapi_namespace + " package"), namespace=multiapi_namespace
+                )
+            )
             for version in apilist:
-                rst_file.write("   {namespace}.{version}\n".format(
-                    namespace=multiapi_namespace,
-                    version=version))
+                rst_file.write("   {namespace}.{version}\n".format(namespace=multiapi_namespace, version=version))
         package_list_path.append(rst_path)
 
-
     # now handle the packages from data plane that we want to be present properly sorted in the list of packages
     for package in config.keys():
-        if 'manually_generated' in config[package] and config[package]['manually_generated'] == True:
-            package_list_path.append(rst_path_template.format(package.replace('-','.')))
+        if "manually_generated" in config[package] and config[package]["manually_generated"] == True:
+            package_list_path.append(rst_path_template.format(package.replace("-", ".")))
 
     package_list_path.sort()
-    with Path(GENERATED_PACKAGES_LIST_FILE).open('w') as generate_file_list_fd:
-        lines_to_write = "\n".join(["  "+package for package in package_list_path])
+    with Path(GENERATED_PACKAGES_LIST_FILE).open("w") as generate_file_list_fd:
+        lines_to_write = "\n".join(["  " + package for package in package_list_path])
         generate_file_list_fd.write(RST_AUTODOC_TOCTREE.format(generated_packages=lines_to_write))
 
+
 def main():
-    parser = argparse.ArgumentParser(
-        description='Generate documentation file.'
+    parser = argparse.ArgumentParser(description="Generate sphinx api stubs for one or multiple management packages.")
+    parser.add_argument(
+        "--project",
+        "-p",
+        dest="project",
+        help="Want to target a specific management package? Pass the targeted package root to this argument.",
+    )
+    parser.add_argument(
+        "--config",
+        "-c",
+        dest="config_path",
+        default=CONFIG_FILE,
+        help="The JSON configuration format path [default: %(default)s]",
+    )
+    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbosity in INFO mode")
+    parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode")
+    parser.add_argument(
+        "--out", "-o", dest="output_directory", help="The final resting place for the generated sphinx source files."
     )
-    parser.add_argument('--project', '-p',
-                        dest='project', action='append',
-                        help='Select a specific project. Do all by default. You can use a substring for several projects.')
-    parser.add_argument('--config', '-c',
-                        dest='config_path', default=CONFIG_FILE,
-                        help='The JSON configuration format path [default: %(default)s]')
-    parser.add_argument("-v", "--verbose",
-                        dest="verbose", action="store_true",
-                        help="Verbosity in INFO mode")
-    parser.add_argument("--debug",
-                        dest="debug", action="store_true",
-                        help="Verbosity in DEBUG mode")
-    parser.add_argument("--out", "-o",
-                        dest="output_directory", 
-                        help="The final resting place for the generated sphinx source files.")
 
     args = parser.parse_args()
 
@@ -184,5 +215,6 @@ def main():
 
     generate_doc(args.config_path, args.output_directory, args.project)
 
+
 if __name__ == "__main__":
     main()

eng/tox/run_sphinx_apidoc.py

@@ -66,15 +66,15 @@ def sphinx_apidoc(working_directory):
         )
         exit(1)
 
-def mgmt_apidoc(working_directory, namespace):
+def mgmt_apidoc(working_directory: str, target_folder: str):
     command_array = [
         sys.executable,
         generate_mgmt_script,
         "-p",
-        namespace,
+        target_folder,
         "-o",
         working_directory,
-        "--verbose"
+        "--verbose",
         ]
 
     try:
@@ -122,7 +122,7 @@ def mgmt_apidoc(working_directory, namespace):
 
     if should_build_docs(pkg_details.name):
         if is_mgmt_package(pkg_details.name):
-            mgmt_apidoc(output_directory, pkg_details.namespace)
+            mgmt_apidoc(output_directory, pkg_details.folder)
         else:
             sphinx_apidoc(args.working_directory)
     else: