Add Optional Environment (Azure#32332)

* updates to eng/ci_tools and eng/test_tools.txt to allow necessary additions for compatibility
* added new namespace to azure-sdk-tools: ci_tools.scenario.
  * ci_tools.scenario is used to emplace any code that actually does the action of installing a dependency, replacing requirements, building on the fly, etc. This means that create_package_and_install and replace_dev_reqs both are moved into this module
* created tox environment 'optional'. This tox environment utilizes array items of pyproject.toml config item 'tool.azure-sdk-build.optional' to install requirements for specific test scenarios before invoking tests against them. These tests are run within a custom venv for each optional configuration item in pyproject.toml, location .tox/optional/<envname>/
  * the new 'optional' environment is enhanced to allow users to pass in the target environment EG: 'tox -e optional --root . -c ../../../eng/tox/tox.ini -- --optional no_requests'

Co-authored-by: Laurent Mazuel <laurent.mazuel@gmail.com>
Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -59,6 +59,7 @@ conda/assembly/
 conda/assembled/
 conda/downloaded/
 conda/conda-env/
+scenario_*.txt
 
 # tox environment folders
 .tox/

.vscode/cspell.json

@@ -127,6 +127,7 @@
     "conda/conda-releaselogs/azure-mgmt.md"
   ],
   "words": [
+    "spinup",
     "cibuildwheel",
     "aoai",
     "pyprojects",

eng/tox/create_dependencies_and_install.py

@@ -11,7 +11,7 @@
 import os
 import logging
 from ci_tools.variables import in_ci
-from ci_tools.functions import build_and_install_dev_reqs
+from ci_tools.scenario.generation import build_and_install_dev_reqs
 logging.getLogger().setLevel(logging.INFO)
 
 

eng/tox/create_package_and_install.py

@@ -9,83 +9,12 @@
 # it should be executed from tox with `{toxenvdir}/python` to ensure that the package
 # can be successfully tested from within a tox environment.
 
-from subprocess import check_call, CalledProcessError
 import argparse
-import os
 import logging
-import sys
-import glob
-import shutil
-
-from tox_helper_tasks import get_pip_list_output
-from ci_tools.parsing import ParsedSetup, parse_require
-from ci_tools.build import create_package
-from ci_tools.functions import get_package_from_repo, find_whl, find_sdist, discover_prebuilt_package
 
 logging.getLogger().setLevel(logging.INFO)
 
-from ci_tools.parsing import ParsedSetup
-
-
-def cleanup_build_artifacts(build_folder):
-    # clean up egginfo
-    results = glob.glob(os.path.join(build_folder, "*.egg-info"))
-
-    if results:
-        print(results[0])
-        shutil.rmtree(results[0])
-
-    # clean up build results
-    build_path = os.path.join(build_folder, "build")
-    if os.path.exists(build_path):
-        shutil.rmtree(build_path)
-
-
-def discover_packages(setuppy_path, args):
-    packages = []
-    if os.getenv("PREBUILT_WHEEL_DIR") is not None and not args.force_create:
-        packages = discover_prebuilt_package(os.getenv("PREBUILT_WHEEL_DIR"), setuppy_path, args.package_type)
-        pkg = ParsedSetup.from_path(setuppy_path)
-
-        if not packages:
-            logging.error(
-                "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
-                    os.getenv("PREBUILT_WHEEL_DIR"), pkg.name, pkg.version
-                )
-            )
-            exit(1)
-    else:
-        packages = build_and_discover_package(
-            setuppy_path,
-            args.distribution_directory,
-            args.target_setup,
-            args.package_type,
-        )
-
-    return packages
-
-
-def in_ci():
-    return os.getenv("TF_BUILD", False)
-
-
-def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_type):
-    if package_type == "wheel":
-        create_package(setuppy_path, dist_dir, enable_sdist=False)
-    else:
-        create_package(setuppy_path, dist_dir, enable_wheel=False)
-
-    prebuilt_packages = [
-        f
-        for f in os.listdir(args.distribution_directory)
-        if f.endswith(".whl" if package_type == "wheel" else ".tar.gz")
-    ]
-
-    if not in_ci():
-        logging.info("Cleaning up build directories and files")
-        cleanup_build_artifacts(target_setup)
-    return prebuilt_packages
-
+from ci_tools.scenario.generation import create_package_and_install
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -119,19 +48,22 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
         "--cache-dir",
         dest="cache_dir",
         help="Location that, if present, will be used as the pip cache directory.",
+        default=None
     )
 
     parser.add_argument(
         "-w",
         "--work-dir",
         dest="work_dir",
         help="Location that, if present, will be used as working directory to run pip install.",
+        default=None
     )
 
     parser.add_argument(
         "--force-create",
         dest="force_create",
         help="Force recreate whl even if it is prebuilt",
+        default=False
     )
 
     parser.add_argument(
@@ -150,117 +82,16 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
 
     args = parser.parse_args()
 
-    commands_options = []
-    built_pkg_path = ""
-    setup_py_path = os.path.join(args.target_setup, "setup.py")
-    additional_downloaded_reqs = []
-
-    if not os.path.exists(args.distribution_directory):
-        os.mkdir(args.distribution_directory)
-
-    tmp_dl_folder = os.path.join(args.distribution_directory, "dl")
-    if not os.path.exists(tmp_dl_folder):
-        os.mkdir(tmp_dl_folder)
-
-    # preview version is enabled when installing dev build so pip will install dev build version from devpos feed
-    if os.getenv("SetDevVersion", "false") == "true":
-        commands_options.append("--pre")
-
-    if args.cache_dir:
-        commands_options.extend(["--cache-dir", args.cache_dir])
-
-    discovered_packages = discover_packages(setup_py_path, args)
-
-    if args.skip_install:
-        logging.info("Flag to skip install whl is passed. Skipping package installation")
-    else:
-        for built_package in discovered_packages:
-            if os.getenv("PREBUILT_WHEEL_DIR") is not None and not args.force_create:
-                # find the prebuilt package in the set of prebuilt wheels
-                package_path = os.path.join(os.environ["PREBUILT_WHEEL_DIR"], built_package)
-                if os.path.isfile(package_path):
-                    built_pkg_path = package_path
-                    logging.info("Installing {w} from directory".format(w=built_package))
-                # it does't exist, so we need to error out
-                else:
-                    logging.error("{w} not present in the prebuilt package directory. Exiting.".format(w=built_package))
-                    exit(1)
-            else:
-                built_pkg_path = os.path.abspath(os.path.join(args.distribution_directory, built_package))
-                logging.info("Installing {w} from fresh built package.".format(w=built_package))
-
-            if not args.pre_download_disabled:
-                requirements = ParsedSetup.from_path(
-                    os.path.join(os.path.abspath(args.target_setup), "setup.py")
-                ).requires
-                azure_requirements = [req.split(";")[0] for req in requirements if req.startswith("azure-")]
-
-                if azure_requirements:
-                    logging.info(
-                        "Found {} azure requirement(s): {}".format(len(azure_requirements), azure_requirements)
-                    )
-
-                    download_command = [
-                        sys.executable,
-                        "-m",
-                        "pip",
-                        "download",
-                        "-d",
-                        tmp_dl_folder,
-                        "--no-deps",
-                    ]
-
-                    installation_additions = []
-
-                    # only download a package if the requirement is not already met, so walk across
-                    # direct install_requires
-                    for req in azure_requirements:
-                        addition_necessary = True
-                        # get all installed packages
-                        installed_pkgs = get_pip_list_output()
-
-                        # parse the specifier
-                        req_name, req_specifier = parse_require(req)
-
-                        # if we have the package already present...
-                        if req_name in installed_pkgs:
-                            # if there is no specifier for the requirement, we can ignore it
-                            if req_specifier is None:
-                                addition_necessary = False
-
-                            # ...do we need to install the new version? if the existing specifier matches, we're fine
-                            if req_specifier is not None and installed_pkgs[req_name] in req_specifier:
-                                addition_necessary = False
-
-                        if addition_necessary:
-                            # we only want to add an additional rec for download if it actually exists
-                            # in the upstream feed (either dev or pypi)
-                            # if it doesn't, we should just install the relative dep if its an azure package
-                            installation_additions.append(req)
-
-                    if installation_additions:
-                        non_present_reqs = []
-                        for addition in installation_additions:
-                            try:
-                                check_call(
-                                    download_command + [addition] + commands_options,
-                                    env=dict(os.environ, PIP_EXTRA_INDEX_URL=""),
-                                )
-                            except CalledProcessError as e:
-                                req_name, req_specifier = parse_require(addition)
-                                non_present_reqs.append(req_name)
-
-                        additional_downloaded_reqs = [
-                            os.path.abspath(os.path.join(tmp_dl_folder, pth)) for pth in os.listdir(tmp_dl_folder)
-                        ] + [get_package_from_repo(relative_req).folder for relative_req in non_present_reqs]
+    create_package_and_install(
+        distribution_directory=args.distribution_directory,
+        target_setup=args.target_setup,
+        skip_install=args.skip_install,
+        cache_dir=args.cache_dir,
+        work_dir=args.work_dir,
+        force_create=args.force_create,
+        package_type=args.package_type,
+        pre_download_disabled=args.pre_download_disabled,
+    )
 
-            commands = [sys.executable, "-m", "pip", "install", built_pkg_path]
-            commands.extend(additional_downloaded_reqs)
-            commands.extend(commands_options)
 
-            if args.work_dir and os.path.exists(args.work_dir):
-                logging.info("Executing command from {0}:{1}".format(args.work_dir, commands))
-                check_call(commands, cwd=args.work_dir)
-            else:
-                check_call(commands)
-            logging.info("Installed {w}".format(w=built_package))
+    

eng/tox/run_optional.py

@@ -0,0 +1,28 @@
+import argparse
+from ci_tools.scenario.generation import prepare_and_test_optional
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="""This entrypoint provides automatic invocation of the 'optional' requirements for a given package. View the pyproject.toml within the targeted package folder to see configuration.""",
+    )
+
+    parser.add_argument("-t", "--target", dest="target", help="The target package path", required=True)
+
+    parser.add_argument(
+        "-o",
+        "--optional",
+        dest="optional",
+        help="The target environment. If not provided, all optional environments will be run.",
+        required=False,
+    )
+
+    parser.add_argument(
+        "--temp",
+        dest="temp_dir",
+        help="The temp directory this script will work in.",
+        required=False,
+    )
+
+    args, _ = parser.parse_known_args()
+    exit(prepare_and_test_optional(mapped_args=args))

eng/tox/tox.ini

@@ -539,6 +539,7 @@ deps=
 commands=
   black --config {repository_root}/eng/black-pyproject.toml {posargs}
 
+
 [testenv:generate]
 description=Regenerate the code
 skip_install=true
@@ -547,3 +548,16 @@ deps =
   tomli==2.0.1
 commands=
   python -m packaging_tools.generate_client
+
+
+[testenv:optional]
+skipsdist = true
+skip_install = true
+usedevelop = true
+changedir = {toxinidir}
+setenv =
+  {[testenv]setenv}
+  PROXY_URL=http://localhost:5018
+commands =
+    {envbindir}/python -m pip install {toxinidir}/../../../tools/azure-sdk-tools[build]
+    python {repository_root}/eng/tox/run_optional.py -t {toxinidir} --temp={envtmpdir} {posargs}

eng/tox/tox_helper_tasks.py

@@ -19,31 +19,6 @@
 
 logging.getLogger().setLevel(logging.INFO)
 
-
-def get_pip_list_output():
-    """Uses the invoking python executable to get the output from pip list."""
-    out = subprocess.Popen(
-        [sys.executable, "-m", "pip", "list", "--disable-pip-version-check", "--format", "freeze"],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-
-    stdout, stderr = out.communicate()
-
-    collected_output = {}
-
-    if stdout and (stderr is None):
-        # this should be compatible with py27 https://docs.python.org/2.7/library/stdtypes.html#str.decode
-        for line in stdout.decode("utf-8").split(os.linesep)[2:]:
-            if line:
-                package, version = re.split("==", line)
-                collected_output[package] = version
-    else:
-        raise Exception(stderr)
-
-    return collected_output
-
-
 def unzip_sdist_to_directory(containing_folder: str) -> str:
     zips = glob.glob(os.path.join(containing_folder, "*.zip"))
 

scripts/devops_tasks/tox_harness.py

@@ -18,8 +18,9 @@
 from ci_tools.variables import in_ci
 from ci_tools.environment_exclusions import filter_tox_environment_string
 from ci_tools.ci_interactions import output_ci_warning
-from ci_tools.functions import build_whl_for_req, cleanup_directory, replace_dev_reqs
-from pkg_resources import parse_requirements
+from ci_tools.scenario.generation import replace_dev_reqs
+from ci_tools.functions import cleanup_directory
+from pkg_resources import parse_requirements, RequirementParseError
 import logging
 
 logging.getLogger().setLevel(logging.INFO)

sdk/core/azure-core/pyproject.toml

@@ -7,4 +7,16 @@ strict_sphinx = true
 # For test environments or static checks where a check should be run by default, not explicitly disabling will enable the check.
 # pylint is enabled by default, so there is no reason for a pylint = true in every pyproject.toml.
 #
-# For newly added checks that are not enabled by default, packages should opt IN by "<check> = true".
+# For newly added checks that are not enabled by default, packages should opt IN by "<check> = true".
+
+[[tool.azure-sdk-build.optional]]
+name = "no_requests"
+install = []
+uninstall = ["requests"]
+additional_pytest_args = ["--ignore-glob='*_async.py'"]
+
+[[tool.azure-sdk-build.optional]]
+name = "no_aiohttp"
+install = []
+uninstall = ["aiohttp"]
+additional_pytest_args = ["-k", "_async.py"]