Enforce proper usage of py.typed file (Azure#24409)

*update verify_sdist.py to also validate the configuration of the package WRT py.typed. Enforces setup.py config, MANIFEST.in config, and checks the existence of a py.typed file at the correct folder depth.

Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com>

Author: scbedd

eng/tox/create_package_and_install.py

@@ -57,7 +57,7 @@ def discover_packages(setuppy_path, args):
 
 def discover_prebuilt_package(dist_directory, setuppy_path, package_type):
     packages = []
-    pkg_name, _, version = get_package_details(setuppy_path)
+    pkg_name, _, version, _, _ = get_package_details(setuppy_path)
     if package_type == "wheel":
         prebuilt_package = find_whl(dist_directory, pkg_name, version)
     else:

eng/tox/import_all.py

@@ -41,7 +41,7 @@ def should_run_import_all(package_name):
 
     # get target package name from target package path
     pkg_dir = os.path.abspath(args.target_package)
-    package_name, namespace, _ = get_package_details(os.path.join(pkg_dir, 'setup.py'))
+    package_name, namespace, _, _, _ = get_package_details(os.path.join(pkg_dir, 'setup.py'))
 
     if should_run_import_all(package_name):
         # import all modules from current package

eng/tox/install_dev_build_dependency.py

@@ -120,7 +120,7 @@ def install_dev_build_packages(pkg_name_to_exclude):
     if args.target_package:
         # get target package name from target package path
         pkg_dir = path.abspath(args.target_package)
-        pkg_name, _, ver = get_package_details(path.join(pkg_dir, "setup.py"))
+        pkg_name, _, ver, _, _ = get_package_details(path.join(pkg_dir, "setup.py"))
         install_dev_build_packages(pkg_name)
 
 

eng/tox/prep_sphinx_env.py

@@ -138,7 +138,7 @@ def write_version(site_folder, version):
     args = parser.parse_args()
 
     package_path = os.path.abspath(args.target_package)
-    package_name, namespace, package_version = get_package_details(
+    package_name, namespace, package_version, _, _ = get_package_details(
         os.path.join(package_path, "setup.py")
     )
 

eng/tox/run_apistubgen.py

@@ -18,7 +18,7 @@
 
 def get_package_wheel_path(pkg_root):
     # parse setup.py to get package name and version
-    pkg_name, _, version = get_package_details(os.path.join(pkg_root, "setup.py"))
+    pkg_name, _, version, _, _ = get_package_details(os.path.join(pkg_root, "setup.py"))
     # Check if wheel is already built and available for current package
     prebuilt_dir = os.getenv("PREBUILT_WHEEL_DIR")
     if prebuilt_dir:

eng/tox/run_pylint.py

@@ -43,7 +43,7 @@
 
 
     pkg_dir = os.path.abspath(args.target_package)
-    package_name, namespace, ver = get_package_details(os.path.join(pkg_dir, "setup.py"))
+    package_name, namespace, ver, _, _ = get_package_details(os.path.join(pkg_dir, "setup.py"))
 
     top_level_module = namespace.split('.')[0]
 

eng/tox/run_sphinx_apidoc.py

@@ -117,7 +117,7 @@ def mgmt_apidoc(working_directory, namespace):
     package_dir = os.path.abspath(args.package_root)
     output_directory = os.path.join(target_dir, "unzipped/docgen")
 
-    pkg_name, namespace, pkg_version = get_package_details(os.path.join(package_dir, 'setup.py'))
+    pkg_name, namespace, pkg_version, _, _ = get_package_details(os.path.join(package_dir, 'setup.py'))
 
     if should_build_docs(pkg_name):
         if is_mgmt_package(pkg_name):

eng/tox/run_sphinx_build.py

@@ -107,7 +107,7 @@ def sphinx_build(target_dir, output_dir):
     target_dir = os.path.abspath(args.working_directory)
     package_dir = os.path.abspath(args.package_root)
 
-    package_name, _, pkg_version = get_package_details(os.path.join(package_dir, 'setup.py'))
+    package_name, _, pkg_version, _, _ = get_package_details(os.path.join(package_dir, 'setup.py'))
 
     if should_build_docs(package_name):
         sphinx_build(target_dir, output_dir)

eng/tox/tox_helper_tasks.py

@@ -60,13 +60,22 @@ def setup(*args, **kwargs):
     package_name = kwargs["name"]
     # default namespace for the package
     name_space = package_name.replace("-", ".")
+
+    package_data = None
+    if "package_data" in kwargs:
+        package_data = kwargs["package_data"]
+
+    include_package_data = None
+    if "include_package_data" in kwargs:
+        include_package_data = kwargs["include_package_data"]
+
     if "packages" in kwargs.keys():
         packages = kwargs["packages"]
         if packages:
             name_space = packages[0]
             logging.info("Namespaces found for package {0}: {1}".format(package_name, packages))
 
-    return package_name, name_space, kwargs["version"]
+    return package_name, name_space, kwargs["version"], package_data, include_package_data
 
 
 def parse_req(req):

eng/tox/verify_sdist.py

@@ -11,28 +11,34 @@
 import os
 import glob
 import shutil
+from unicodedata import name
+from xmlrpc.client import Boolean
 from tox_helper_tasks import (
     get_package_details,
     unzip_file_to_directory,
 )
 from verify_whl import cleanup, should_verify_package
+from typing import List, Mapping, Any
 
 logging.getLogger().setLevel(logging.INFO)
 
 ALLOWED_ROOT_DIRECTORIES = ["azure", "tests", "samples", "examples"]
 
+EXCLUDED_PYTYPE_PACKAGES = ["azure-keyvault", "azure", "azure-common"]
 
-def get_root_directories_in_source(package_dir):
-    # find all allowed directories in source path
-    source_folders = [
-        d
-        for d in os.listdir(package_dir)
-        if os.path.isdir(d) and d in ALLOWED_ROOT_DIRECTORIES
-    ]
+
+def get_root_directories_in_source(package_dir: str) -> List[str]:
+    """
+    Find all allowed directories in source path.
+    """
+    source_folders = [d for d in os.listdir(package_dir) if os.path.isdir(d) and d in ALLOWED_ROOT_DIRECTORIES]
     return source_folders
 
 
-def get_root_directories_in_sdist(dist_dir, version):
+def get_root_directories_in_sdist(dist_dir: str, version: str) -> List[str]:
+    """
+    Given an unzipped sdist directory, extract which directories are present.
+    """
     # find sdist zip file
     # extract sdist and find list of directories in sdist
     path_to_zip = glob.glob(os.path.join(dist_dir, "*{}*.zip".format(version)))[0]
@@ -44,8 +50,10 @@ def get_root_directories_in_sdist(dist_dir, version):
     return sdist_folders
 
 
-def verify_sdist(package_dir, dist_dir, version):
-    # This function compares the root directories in source against root directories sdist
+def verify_sdist(package_dir: str, dist_dir: str, version: str) -> bool:
+    """
+    Compares the root directories in source against root directories present within a sdist.
+    """
 
     source_folders = get_root_directories_in_source(package_dir)
     sdist_folders = get_root_directories_in_sdist(dist_dir, version)
@@ -63,9 +71,49 @@ def verify_sdist(package_dir, dist_dir, version):
         return True
 
 
+def verify_sdist_pytyped(
+    pkg_dir: str, namespace: str, package_metadata: Mapping[str, Any], include_package_data: bool
+) -> bool:
+    """
+    Takes a package directory and ensures that the setup.py within is correctly configured for py.typed files.
+    """
+    result = True
+    manifest_location = os.path.join(pkg_dir, "MANIFEST.in")
+
+    if include_package_data is None or False:
+        logging.info(
+            "Ensure that the setup.py present in directory {} has kwarg 'include_package_data' defined and set to 'True'."
+        )
+        result = False
+
+    if package_metadata:
+        if not any([key for key in package_metadata if "py.typed" in str(package_metadata[key])]):
+            logging.info(
+                "At least one value in the package_metadata map should include a reference to the py.typed file."
+            )
+            result = False
+
+    if os.path.exists(manifest_location):
+        with open(manifest_location, "r") as f:
+            lines = f.readlines()
+            result = any([include for include in lines if "py.typed" in include])
+
+            if not result:
+                logging.info("Ensure that the MANIFEST.in includes at least one path that leads to a py.typed file.")
+
+    pytyped_file_path = os.path.join(pkg_dir, *namespace.split("."), "py.typed")
+    if not os.path.exists(pytyped_file_path):
+        logging.info(
+            "The py.typed file must exist in the base namespace for your package. Traditionally this would mean the furthest depth, EG 'azure/storage/blob/py.typed'."
+        )
+        result = False
+
+    return result
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="Verify directories included in whl and contents in manifest file"
+        description="Verify directories included in sdist and contents in manifest file. Also ensures that py.typed configuration is correct within the setup.py."
     )
 
     parser.add_argument(
@@ -88,7 +136,9 @@ def verify_sdist(package_dir, dist_dir, version):
 
     # get target package name from target package path
     pkg_dir = os.path.abspath(args.target_package)
-    pkg_name, _, ver = get_package_details(os.path.join(pkg_dir, "setup.py"))
+    pkg_name, namespace, ver, package_data, include_package_data = get_package_details(
+        os.path.join(pkg_dir, "setup.py")
+    )
 
     if should_verify_package(pkg_name):
         logging.info("Verifying sdist for package [%s]", pkg_name)
@@ -97,3 +147,11 @@ def verify_sdist(package_dir, dist_dir, version):
         else:
             logging.info("Failed to verify sdist for package [%s]", pkg_name)
             exit(1)
+
+    if pkg_name not in EXCLUDED_PYTYPE_PACKAGES and "-nspkg" not in pkg_name:
+        logging.info("Verifying presence of py.typed: [%s]", pkg_name)
+        if verify_sdist_pytyped(pkg_dir, namespace, package_data, include_package_data):
+            logging.info("Py.typed setup.py kwargs are set properly: [%s]", pkg_name)
+        else:
+            logging.info("Verified py.typed [%s]. Check messages above.", pkg_name)
+            exit(1)