Adding the ability to skip compatibility filtering (Azure#25813)

* adding the ability to skip the compatibility filter where appropriate

Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com>

Author: scbedd

scripts/devops_tasks/setup_execute_tests.py

@@ -26,7 +26,7 @@
     create_code_coverage_params
 )
 from tox_harness import prep_and_run_tox
-from ci_tools.functions import discover_targeted_packages, filter_packages_by_compatibility_override
+from ci_tools.functions import discover_targeted_packages
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -280,9 +280,8 @@ def execute_global_install_and_test(
         dest="filter_type",
         default='Build',
         help="Filter type to identify eligible packages. for e.g. packages filtered in Build can pass filter type as Build,",
-        choices=['Build', "Docs", "Regression", "Omit_management"]
+        choices=['Build', "Docs", "Regression", "Omit_management", "None"]
     )
-
     parser.add_argument(
         "-d",
         "--dest-dir",
@@ -301,21 +300,23 @@ def execute_global_install_and_test(
     else:
         target_dir = root_dir
 
-    targeted_packages = discover_targeted_packages(args.glob_string, target_dir, "", args.filter_type)
-    compatible_targeted_packages = filter_packages_by_compatibility_override(targeted_packages)
+    if args.filter_type == "None":
+        args.filter_type = "Build"
+        compatibility_filter = False
+    else:
+        compatibility_filter = True
 
-    if targeted_packages != compatible_targeted_packages:
-        logging.info("At least one package incompatible with current platform was detected. Skipping: {}".format(set(targeted_packages) - set(compatible_targeted_packages)))
+    targeted_packages = discover_targeted_packages(args.glob_string, target_dir, "", args.filter_type, compatibility_filter)
 
     extended_pytest_args = []
 
-    if len(compatible_targeted_packages) == 0:
+    if len(targeted_packages) == 0:
         exit(0)
 
     if args.xdist:
         extended_pytest_args.extend(["-n", "8", "--dist=loadscope"])
 
     if args.runtype != "none":
-        execute_global_install_and_test(args, compatible_targeted_packages, extended_pytest_args)
+        execute_global_install_and_test(args, targeted_packages, extended_pytest_args)
     else:
-        prep_and_run_tox(compatible_targeted_packages, args, extended_pytest_args)
+        prep_and_run_tox(targeted_packages, args, extended_pytest_args)

tools/azure-sdk-tools/ci_tools/functions.py

@@ -42,7 +42,7 @@
     "azure-mgmt-core",
 ]
 
-TEST_COMPATIBILITY_MAP = {"azure-core-tracing-opentelemetry": "<3.10"}
+TEST_COMPATIBILITY_MAP = {}
 
 omit_regression = (
     lambda x: "nspkg" not in x
@@ -66,13 +66,25 @@
 }
 
 
-def filter_for_compatibility(package_set):
+def filter_for_compatibility(package_set: List[str]) -> List[str]:
+    """
+    This function takes in a set of paths to python packages. It returns the set filtered by compatibility with the currently running python executable.
+    If a package is unsupported by the executable, it will be omitted from the returned list.
+
+    A manual override list of these compatibilities is also supported, but requires code change to enable. Check TEST_COMPATIBILITY_MAP in this same file.
+
+    :param List[str] package_set: a list of paths to python packages. Each item can either be a package folder or a direct path to setup.py.
+    """
     collected_packages = []
     v = sys.version_info
     running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])]))
 
     for pkg in package_set:
         spec_set = SpecifierSet(ParsedSetup.from_path(pkg).python_requires)
+        pkg_specs_override = TEST_COMPATIBILITY_MAP.get(os.path.basename(pkg), None)
+
+        if pkg_specs_override:
+            spec_set = SpecifierSet(pkg_specs_override)
 
         if running_major_version in spec_set:
             collected_packages.append(pkg)
@@ -87,14 +99,6 @@ def compare_python_version(version_spec: str):
     return current_sys_version in spec_set
 
 
-def filter_packages_by_compatibility_override(package_set: List[str], resolve_basename: bool = True) -> List[str]:
-    return [
-        p
-        for p in package_set
-        if compare_python_version(TEST_COMPATIBILITY_MAP.get(os.path.basename(p) if resolve_basename else p, ">=2.7"))
-    ]
-
-
 def str_to_bool(input_string: str) -> bool:
     """
     Takes a boolean string representation and returns a bool type value.
@@ -114,10 +118,17 @@ def discover_targeted_packages(
     target_root_dir: str,
     additional_contains_filter: str = "",
     filter_type: str = "Build",
+    compatibility_filter: bool = True,
 ) -> List[str]:
     """
     During build and test, the set of targeted packages may expand or contract depending on the needs of the invocation.
     This function centralizes business and material requirements and outputs the set of packages that should be targeted.
+
+    :param str glob_string: The basic glob used to query packages within the repo. Defaults to "azure-*"
+    :param str target_root_dir: The root directory in which globbing will begin.
+    :param str additional_contains_filter: Additional filter option. Used when needing to provide one-off filtration that doesn't merit an additional filter_type. Defaults to empty string.
+    :param str filter_type: One a string representing a filter function as a set of options. Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build".
+    :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. If the invoking python executable does not match a found package's specifiers, the package will be omitted. Defaults to True.
     """
     if glob_string:
         individual_globs = glob_string.split(",")
@@ -137,12 +148,14 @@ def discover_targeted_packages(
     # if we have individually queued this specific package, it's obvious that we want to build it specifically
     # in this case, do not honor the omission list
     if len(collected_directories) == 1:
-        pkg_set_ci_filtered = filter_for_compatibility(collected_directories)
+        if compatibility_filter:
+            pkg_set_ci_filtered = filter_for_compatibility(collected_directories)
     # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted
     # packages
     else:
         allowed_package_set = remove_omitted_packages(collected_directories)
-        pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set)
+        if compatibility_filter:
+            pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set)
 
     # Apply filter based on filter type. for e.g. Docs, Regression, Management
     pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered))
@@ -265,7 +278,7 @@ def process_requires(setup_py_path: str):
             #            \d+         <followed by number of digits N>
             #        )?
             #       )?
-            rx = r'{}(((a|b|rc)\d+)?(\.post\d+)?)?'.format(base_version)
+            rx = r"{}(((a|b|rc)\d+)?(\.post\d+)?)?".format(base_version)
             new_req = re.sub(rx, "{}{}1".format(base_version, DEV_BUILD_IDENTIFIER), str(req), flags=re.IGNORECASE)
             logging.info("New requirement for package {0}: {1}".format(pkg_name, new_req))
             requirement_to_update[old_req] = new_req

tools/azure-sdk-tools/devtools_testutils/proxy_fixtures.py

@@ -63,7 +63,6 @@ def sanitize_batch(self, variables: "Dict[str, str]") -> "Dict[str, str]":
         current_values = {variable: self.sanitize(variable, variables[variable]) for variable in variables}
         return current_values
 
-
     def get(self, variable: str) -> str:
         """Returns the value of the specified environment variable in live mode, or the sanitized value in playback.
 

tools/azure-sdk-tools/devtools_testutils/sanitizers.py

@@ -571,11 +571,7 @@ def _send_recording_options_request(parameters: dict, headers: "Optional[dict]"
     if is_live_and_not_recording():
         return
 
-    response = requests.post(
-        f"{PROXY_URL}/Admin/SetRecordingOptions",
-        headers=headers,
-        json=parameters
-    )
+    response = requests.post(f"{PROXY_URL}/Admin/SetRecordingOptions", headers=headers, json=parameters)
     response.raise_for_status()
 
 

tools/azure-sdk-tools/packaging_tools/auto_codegen.py

@@ -8,11 +8,19 @@
     CONFIG_FILE,
 )
 from .generate_sdk import generate
-from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview,
-                             format_samples, gen_dpg, dpg_relative_folder)
+from .generate_utils import (
+    get_package_names,
+    init_new_service,
+    update_servicemetadata,
+    judge_tag_preview,
+    format_samples,
+    gen_dpg,
+    dpg_relative_folder,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
+
 def main(generate_input, generate_output):
     with open(generate_input, "r") as reader:
         data = json.load(reader)
@@ -25,11 +33,18 @@ def main(generate_input, generate_output):
     for input_readme in data["relatedReadmeMdFiles"]:
         relative_path_readme = str(Path(spec_folder, input_readme))
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
-        if 'resource-manager' in input_readme:
-            config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True,
-                              python_tag=python_tag)
+        if "resource-manager" in input_readme:
+            config = generate(
+                CONFIG_FILE,
+                sdk_folder,
+                [],
+                relative_path_readme,
+                spec_folder,
+                force_generation=True,
+                python_tag=python_tag,
+            )
         else:
-            config = gen_dpg(input_readme, data.get('autorestConfig', ''), dpg_relative_folder(spec_folder))
+            config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 

tools/azure-sdk-tools/packaging_tools/change_log.py

@@ -33,9 +33,13 @@ def build_md(self):
         if self.features:
             _build_md(sorted(set(self.features), key=self.features.index), "### Features Added", buffer)
         if self.breaking_changes:
-            _build_md(sorted(set(self.breaking_changes), key=self.breaking_changes.index), "### Breaking Changes", buffer)
+            _build_md(
+                sorted(set(self.breaking_changes), key=self.breaking_changes.index), "### Breaking Changes", buffer
+            )
         if not (self.features or self.breaking_changes) and self.optional_features:
-            _build_md(sorted(set(self.optional_features), key=self.optional_features.index), "### Features Added", buffer)
+            _build_md(
+                sorted(set(self.optional_features), key=self.optional_features.index), "### Features Added", buffer
+            )
 
         return "\n".join(buffer).strip()
 

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -188,9 +188,11 @@ def gen_general_namespace(package_name: str) -> str:
 def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str:
     package_name = Path(origin_config["output-folder"]).parts[-1]
     readme_config = gen_basic_config(origin_config, spec_folder)
-    readme_config.update({
-        "namespace": gen_general_namespace(package_name),
-    })
+    readme_config.update(
+        {
+            "namespace": gen_general_namespace(package_name),
+        }
+    )
     readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings")
     return add_config_title(readme_content)
 

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -7,8 +7,14 @@
 
 from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE
 from .generate_sdk import generate
-from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview,
-                             format_samples, gen_dpg)
+from .generate_utils import (
+    get_package_names,
+    init_new_service,
+    update_servicemetadata,
+    judge_tag_preview,
+    format_samples,
+    gen_dpg,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -18,21 +24,22 @@ def main(generate_input, generate_output):
         data = json.load(reader)
         _LOGGER.info(f"auto_package input: {data}")
 
-    spec_folder = re.sub('specification', '', data["specFolder"])
+    spec_folder = re.sub("specification", "", data["specFolder"])
     sdk_folder = "."
     result = {}
-    python_tag = data.get('python_tag')
+    python_tag = data.get("python_tag")
     package_total = set()
 
     input_readme = data["relatedReadmeMdFile"]
     relative_path_readme = str(Path(spec_folder, input_readme))
     _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
 
-    if 'resource-manager' in input_readme:
-        config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True,
-                          python_tag=python_tag)
+    if "resource-manager" in input_readme:
+        config = generate(
+            CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag
+        )
     else:
-        config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder)
+        config = gen_dpg(input_readme, data.get("autorestConfig", ""), spec_folder)
     package_names = get_package_names(sdk_folder)
     _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 

tools/azure-sdk-tools/pypi_tools/pypi.py

@@ -12,6 +12,7 @@ def get_pypi_xmlrpc_client():
 class PyPIClient:
     def __init__(self, host="https://pypi.org"):
         import requests
+
         self._host = host
         self._session = requests.Session()
 

tools/azure-sdk-tools/tests/test_parse_functionality.py

@@ -4,7 +4,8 @@
 import pdb
 from unittest.mock import patch
 
-package_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+package_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+
 
 def test_parse_require():
     test_scenarios = [
@@ -14,15 +15,15 @@ def test_parse_require():
         "msrest==0.6.10",
         "msrest<0.6.10",
         "msrest>0.6.9",
-        "azure-core<2.0.0,>=1.2.2"
+        "azure-core<2.0.0,>=1.2.2",
     ]
 
     for scenario in test_scenarios:
         result = parse_require(scenario)
 
-        assert(result[0] is not None)
-        assert(result[1] is not None)
-        assert(isinstance(result[1], SpecifierSet))
+        assert result[0] is not None
+        assert result[1] is not None
+        assert isinstance(result[1], SpecifierSet)
 
 
 def test_parse_require_with_no_spec():
@@ -31,10 +32,11 @@ def test_parse_require_with_no_spec():
     for scenario in spec_scenarios:
         result = parse_require(scenario)
 
-        assert(result[0] == scenario.replace("_", "-"))
-        assert(result[1] is None)
+        assert result[0] == scenario.replace("_", "-")
+        assert result[1] is None
 
-@patch('ci_tools.parsing.parse_functions.read_setup_py_content')
+
+@patch("ci_tools.parsing.parse_functions.read_setup_py_content")
 def test_sdk_sample_setup(test_patch):
     test_patch.return_value = """
 import re
@@ -94,19 +96,15 @@ def test_sdk_sample_setup(test_patch):
     ],
 )
     """
-    
+
     result = ParsedSetup.from_path(package_root)
 
-    assert(result.name == "azure-core")
-    assert(result.version == "1.21.0")
-    assert(result.python_requires == ">=3.7")
-    assert(result.requires == ['requests>=2.18.4', 'six>=1.11.0', 'typing-extensions>=4.0.1'])
-    assert(result.is_new_sdk == True)
-    assert(result.setup_filename == os.path.join(package_root, "setup.py"))
-    assert(result.namespace == "ci_tools")
-    assert("pytyped" in result.package_data)
-    assert(result.include_package_data == True)
-
-    
-    
-    
+    assert result.name == "azure-core"
+    assert result.version == "1.21.0"
+    assert result.python_requires == ">=3.7"
+    assert result.requires == ["requests>=2.18.4", "six>=1.11.0", "typing-extensions>=4.0.1"]
+    assert result.is_new_sdk == True
+    assert result.setup_filename == os.path.join(package_root, "setup.py")
+    assert result.namespace == "ci_tools"
+    assert "pytyped" in result.package_data
+    assert result.include_package_data == True