Fix azpysdk Sphinx Check and Use in CI (#44144)

* minor bug fix

* unzip

* cut

* fix mgmt apidoc

* use constants for repeated strings

* move should_build_docs

* docstrings

* update yml

Author: JennyPng

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -67,13 +67,17 @@ steps:
       displayName: 'Generate Docs'
       condition: and(succeededOrFailed(), ${{parameters.BuildDocs}})
       inputs:
-        scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
+        scriptPath: 'eng/scripts/dispatch_checks.py'
         arguments: >-
           "$(TargetingString)"
           --service="${{ parameters.ServiceDirectory }}"
-          --toxenv=sphinx
+          --checks="sphinx"
           --wheel_dir="$(Build.ArtifactStagingDirectory)"
-
+      env:
+        TOX_PIP_IMPL: "uv"
+        VIRTUAL_ENV: ""
+        PYTHONHOME: ""
+        
   - ${{ if eq(parameters.RunApiStubGen, 'true') }}:
     - template: /eng/pipelines/templates/steps/run_apistub.yml
       parameters:

eng/tools/azure-sdk-tools/azpysdk/sphinx.py

@@ -3,13 +3,15 @@
 import sys
 import shutil
 import glob
+import zipfile
+import tarfile
 
 from typing import Optional, List
 from subprocess import CalledProcessError, check_call
 from pathlib import Path
 
 from .Check import Check
-from ci_tools.functions import install_into_venv
+from ci_tools.functions import install_into_venv, unzip_file_to_directory
 from ci_tools.scenario.generation import create_package_and_install
 from ci_tools.variables import in_ci, set_envvar_defaults
 from ci_tools.variables import discover_repo_root
@@ -46,10 +48,55 @@
 ci_doc_dir = os.path.join(REPO_ROOT, "_docs")
 sphinx_conf_dir = os.path.join(REPO_ROOT, "doc/sphinx")
 generate_mgmt_script = os.path.join(REPO_ROOT, "doc/sphinx/generate_doc.py")
+UNZIPPPED_DIR_NAME = "unzipped"
+DOCGEN_DIR_NAME = "docgen"
+
+
+def unzip_sdist_to_directory(containing_folder: str) -> str:
+    """
+    Unzips the first .zip or .tar.gz file found in the containing_folder to that same folder.
+
+    :param containing_folder: The folder to search for .zip or .tar.gz files.
+    :return: The path to the directory where the archive was extracted.
+    """
+    zips = glob.glob(os.path.join(containing_folder, "*.zip"))
+
+    if zips:
+        return unzip_file_to_directory(zips[0], containing_folder)
+    else:
+        tars = glob.glob(os.path.join(containing_folder, "*.tar.gz"))
+
+        if tars:
+            return unzip_file_to_directory(tars[0], containing_folder)
+        else:
+            raise FileNotFoundError("No .zip or .tar.gz files found in {}".format(containing_folder))
+
+
+def move_and_rename(source_location: str) -> str:
+    """
+    Moves and renames the extracted sdist folder to a known name.
+
+    :param source_location: The path to the extracted sdist folder.
+    :return: The new path after moving and renaming.
+    """
+    new_location = os.path.join(os.path.dirname(source_location), UNZIPPPED_DIR_NAME)
+
+    if os.path.exists(new_location):
+        shutil.rmtree(new_location)
+
+    os.rename(source_location, new_location)
+    return new_location
 
 
 # env prep helper functions
 def create_index_file(readme_location: str, package_rst: str) -> str:
+    """
+    Create the index file content by combining the readme and package rst reference.
+
+    :param readme_location: The path to the README file.
+    :param package_rst: The package rst reference.
+    :return: The combined index file content.
+    """
     readme_ext = os.path.splitext(readme_location)[1]
 
     output = ""
@@ -65,6 +112,13 @@ def create_index_file(readme_location: str, package_rst: str) -> str:
 
 
 def create_index(doc_folder: str, source_location: str, namespace: str) -> None:
+    """
+    Create the index.md file in the documentation folder.
+
+    :param doc_folder: The path to the documentation folder.
+    :param source_location: The path to the source location.
+    :param namespace: The namespace for the documentation.
+    """
     index_content = ""
 
     package_rst = "{}.rst".format(namespace)
@@ -174,6 +228,10 @@ def run(self, args: argparse.Namespace) -> int:
                 logger.error("This tool requires Python 3.11 or newer. Please upgrade your Python interpreter.")
                 return 1
 
+            if not should_build_docs(package_name):
+                logger.info("Skipping sphinx for {}".format(package_name))
+                continue
+
             self.install_dev_reqs(executable, args, package_dir)
 
             create_package_and_install(
@@ -220,44 +278,36 @@ def run(self, args: argparse.Namespace) -> int:
             logger.info(f"Running sphinx against {package_name}")
 
             # prep env for sphinx
-            doc_folder = os.path.join(staging_directory, "docgen")
             site_folder = os.path.join(package_dir, "website")
+            doc_folder = os.path.join(staging_directory, DOCGEN_DIR_NAME)
 
-            if should_build_docs(package_name):
-                create_index(doc_folder, package_dir, parsed.namespace)
-
-                write_version(site_folder, parsed.version)
-            else:
-                logger.info("Skipping sphinx prep for {}".format(package_name))
+            source_location = move_and_rename(unzip_sdist_to_directory(staging_directory))
+            doc_folder = os.path.join(source_location, DOCGEN_DIR_NAME)
+            create_index(doc_folder, source_location, parsed.namespace)
+            write_version(site_folder, parsed.version)
 
             # run apidoc
-            if should_build_docs(parsed.name):
-                if is_mgmt_package(parsed.name):
-                    results.append(self.mgmt_apidoc(doc_folder, package_dir, executable))
-                else:
-                    results.append(self.sphinx_apidoc(staging_directory, package_dir, parsed.namespace, executable))
+            output_dir = os.path.join(staging_directory, f"{UNZIPPPED_DIR_NAME}/{DOCGEN_DIR_NAME}")
+            if is_mgmt_package(package_name):
+                results.append(self.mgmt_apidoc(output_dir, package_dir, executable))
             else:
-                logger.info("Skipping sphinx source generation for {}".format(parsed.name))
+                results.append(self.sphinx_apidoc(staging_directory, parsed.namespace, executable))
 
             # build
-            if should_build_docs(package_name):
-                # Only data-plane libraries run strict sphinx at the moment
-                fail_on_warning = not is_mgmt_package(package_name)
-                results.append(
-                    # doc_folder = source
-                    # site_folder  = output
-                    self.sphinx_build(package_dir, doc_folder, site_folder, fail_on_warning, executable)
-                )
-
-                if in_ci() or args.in_ci:
-                    move_output_and_compress(site_folder, package_dir, package_name)
-                    if in_analyze_weekly():
-                        from gh_tools.vnext_issue_creator import close_vnext_issue
+            # Only data-plane libraries run strict sphinx at the moment
+            fail_on_warning = not is_mgmt_package(package_name)
+            results.append(
+                # doc_folder = source
+                # site_folder  = output
+                self.sphinx_build(package_dir, doc_folder, site_folder, fail_on_warning, executable)
+            )
 
-                        close_vnext_issue(package_name, "sphinx")
+            if in_ci() or args.in_ci:
+                move_output_and_compress(site_folder, package_dir, package_name)
+                if in_analyze_weekly():
+                    from gh_tools.vnext_issue_creator import close_vnext_issue
 
-            else:
-                logger.info("Skipping sphinx build for {}".format(package_name))
+                    close_vnext_issue(package_name, "sphinx")
 
         return max(results) if results else 0
 
@@ -282,7 +332,9 @@ def sphinx_build(
         try:
             logger.info("Sphinx build command: {}".format(command_array))
 
-            self.run_venv_command(executable, command_array, cwd=package_dir, check=True, append_executable=False)
+            self.run_venv_command(
+                executable, command_array, cwd=package_dir, check=True, append_executable=False, immediately_dump=True
+            )
         except CalledProcessError as e:
             logger.error("sphinx-build failed for path {} exited with error {}".format(target_dir, e.returncode))
             if in_analyze_weekly():
@@ -294,7 +346,6 @@ def sphinx_build(
 
     def mgmt_apidoc(self, output_dir: str, target_folder: str, executable: str) -> int:
         command_array = [
-            executable,
             generate_mgmt_script,
             "-p",
             target_folder,
@@ -306,48 +357,53 @@ def mgmt_apidoc(self, output_dir: str, target_folder: str, executable: str) -> i
         try:
             logger.info("Command to generate management sphinx sources: {}".format(command_array))
 
-            self.run_venv_command(executable, command_array, cwd=target_folder, check=True, append_executable=False)
+            self.run_venv_command(
+                executable, command_array, cwd=target_folder, check=True, append_executable=True, immediately_dump=True
+            )
         except CalledProcessError as e:
             logger.error("script failed for path {} exited with error {}".format(output_dir, e.returncode))
             return 1
         return 0
 
-    def sphinx_apidoc(self, output_dir: str, target_dir: str, namespace: str, executable: str) -> int:
-        working_doc_folder = os.path.join(output_dir, "doc")
+    def sphinx_apidoc(self, target_dir: str, namespace: str, executable: str) -> int:
+        working_doc_folder = os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/doc")
         command_array = [
             "sphinx-apidoc",
             "--no-toc",
             "--module-first",
             "-o",
-            os.path.join(output_dir, "docgen"),  # This is the output folder
-            os.path.join(target_dir, ""),  # This is the input folder
-            os.path.join(target_dir, "test*"),  # This argument and below are "exclude" directory arguments
-            os.path.join(target_dir, "example*"),
-            os.path.join(target_dir, "sample*"),
-            os.path.join(target_dir, "setup.py"),
-            os.path.join(target_dir, "conftest.py"),
+            os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/{DOCGEN_DIR_NAME}"),  # This is the output folder
+            os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/"),  # This is the input folder
+            os.path.join(
+                target_dir, f"{UNZIPPPED_DIR_NAME}/test*"
+            ),  # This argument and below are "exclude" directory arguments
+            os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/example*"),
+            os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/sample*"),
+            os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/setup.py"),
         ]
 
         try:
             # if a `doc` folder exists, just leverage the sphinx sources found therein.
             if os.path.exists(working_doc_folder):
                 logger.info("Copying files into sphinx source folder.")
-                copy_existing_docs(working_doc_folder, os.path.join(output_dir, "docgen"))
+                copy_existing_docs(
+                    working_doc_folder, os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/{DOCGEN_DIR_NAME}")
+                )
 
             # otherwise, we will run sphinx-apidoc to generate the sources
             else:
                 logger.info("Sphinx api-doc command: {}".format(command_array))
                 self.run_venv_command(executable, command_array, cwd=target_dir, check=True, append_executable=False)
                 # We need to clean "azure.rst", and other RST before the main namespaces, as they are never
                 # used and will log as a warning later by sphinx-build, which is blocking strict_sphinx
-                base_path = Path(os.path.join(output_dir, "docgen/"))
+                base_path = Path(os.path.join(target_dir, f"{UNZIPPPED_DIR_NAME}/{DOCGEN_DIR_NAME}/"))
                 namespace = namespace.rpartition(".")[0]
                 while namespace:
                     rst_file_to_delete = base_path / f"{namespace}.rst"
                     logger.info(f"Removing {rst_file_to_delete}")
                     rst_file_to_delete.unlink(missing_ok=True)
                     namespace = namespace.rpartition(".")[0]
         except CalledProcessError as e:
-            logger.error("sphinx-apidoc failed for path {} exited with error {}".format(output_dir, e.returncode))
+            logger.error("sphinx-apidoc failed for path {} exited with error {}".format(target_dir, e.returncode))
             return 1
         return 0

eng/tools/azure-sdk-tools/ci_tools/functions.py

@@ -92,6 +92,13 @@
 
 
 def unzip_file_to_directory(path_to_zip_file: str, extract_location: str) -> str:
+    """
+    Unzips a zip or tar.gz file to a given location.
+
+    :param path_to_zip_file: The path to the zip or tar.gz file.
+    :param extract_location: The directory where the contents will be extracted.
+    :return: The path to the directory where the archive was extracted.
+    """
     if path_to_zip_file.endswith(".zip"):
         with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref:
             zip_ref.extractall(extract_location)