Enabling Python 3.10 in build matrix (Azure#21063)

* adding py310 on ubuntu 2004 to the test matrix
* pin cffi to prerelease version on py 3.10
* in test_tools.txt and ci_tools.txt, update dependencies where py3.10 necessitates
* add ability to skip a package for a specific python version. Doing this because opentelemetry-api is a hard dependency and there is no timeline on a fix.

Author: scbedd

eng/ci_tools.txt

@@ -3,15 +3,15 @@ cryptography==3.1
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
-wheel==0.34.2
+wheel==0.37.0
 Jinja2==2.11.2
 packaging==20.4
 tox==3.15.0
 tox-monorepo==0.1.2
 twine==1.15.0; python_version == '2.7' or python_version == '3.5'
 twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
-readme-renderer[md]==25.0
+readme-renderer[md]==25.0; python_version < '3.10'
 doc-warden==0.7.1
 # we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
@@ -22,6 +22,9 @@ pip==20.3
 black==21.6b0; python_version >= '3.6'
 wrapt<=1.12.1; python_version == '2.7'
 
+# python 3.10 pinned packages
+cffi==1.15.0rc2; python_version >= '3.10'
+
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 pyOpenSSL==19.1.0

eng/pipelines/templates/stages/platform-matrix-cryptography-dependency.json

@@ -23,6 +23,17 @@
       },
       "PythonVersion": "3.6",
       "CoverageArg": "--disablecov"
+    },
+    {
+      "Config": {
+        "Ubuntu2004_310": {
+          "OSVmImage": "MMSUbuntu20.04",
+          "Pool": "azsdk-pool-mms-ubuntu-2004-general",
+          "PythonVersion": "3.10.0",
+          "CoverageArg": "--disablecov",
+          "TestSamples": "false"
+        }
+      }
     }
   ]
 }

eng/pipelines/templates/stages/platform-matrix.json

@@ -25,6 +25,17 @@
           "TestSamples": "false"
         }
       }
+    },
+    {
+      "Config": {
+        "Ubuntu2004_310": {
+          "OSVmImage": "MMSUbuntu20.04",
+          "Pool": "azsdk-pool-mms-ubuntu-2004-general",
+          "PythonVersion": "3.10.0",
+          "CoverageArg": "--disablecov",
+          "TestSamples": "false"
+        }
+      }
     }
   ]
 }

eng/test_tools.txt

@@ -1,10 +1,7 @@
 pip==20.3
 
-# manually published packages
-https://docsupport.blob.core.windows.net/repackaged/cffi-1.14.6-cp310-cp310-win_amd64.whl; sys_platform=='win32' and python_version >= '3.10'
-https://docsupport.blob.core.windows.net/repackaged/multidict-5.1.0-cp310-cp310-win_amd64.whl; sys_platform=='win32' and python_version >= '3.10'
-https://docsupport.blob.core.windows.net/repackaged/pywin32-301.1-cp310-cp310-win_amd64.whl; sys_platform=='win32' and python_version >= '3.10'
-https://docsupport.blob.core.windows.net/repackaged/yarl-1.6.3-cp310-cp310-win_amd64.whl; sys_platform=='win32' and python_version >= '3.10'
+# python 3.10 pinned packages
+cffi==1.15.0rc2; python_version >= '3.10'
 
 # requirements leveraged by ci for testing
 pytest==4.6.9; python_version == '2.7'
@@ -29,7 +26,7 @@ ConfigArgParse==1.2.3
 six==1.14.0
 pyyaml==5.3.1
 packaging==20.4
-wheel==0.34.2
+wheel==0.37.0
 Jinja2==2.11.2
 
 # Locking pylint and required packages

scripts/devops_tasks/common_tasks.py

@@ -30,7 +30,6 @@
 from packaging.version import Version
 from packaging.version import parse
 
-
 DEV_REQ_FILE = "dev_requirements.txt"
 NEW_DEV_REQ_FILE = "new_dev_requirements.txt"
 NEW_REQ_PACKAGES = ["azure-core", "azure-mgmt-core"]
@@ -65,6 +64,10 @@
     "azure-mgmt-core",
 ]
 
+TEST_COMPATIBILITY_MAP = {
+    "azure-core-tracing-opentelemetry": "<3.10"
+}
+
 omit_regression = (
     lambda x: "nspkg" not in x
     and "mgmt" not in x
@@ -220,6 +223,17 @@ def compare_python_version(version_spec):
     return current_sys_version in spec_set
 
 
+def filter_packages_by_compatibility_override(package_set, resolve_basename=True):
+    return [
+        p
+        for p in package_set
+        if compare_python_version(
+            TEST_COMPATIBILITY_MAP.get(
+                os.path.basename(p) if resolve_basename else p, ">=2.7"
+            )
+        )
+    ]
+
 # this function is where a glob string gets translated to a list of packages
 # It is called by both BUILD (package) and TEST. In the future, this function will be the central location
 # for handling targeting of release packages

scripts/devops_tasks/install_python_version.py

@@ -19,7 +19,7 @@
 MANIFEST_LOCATION = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json"
 
 MAX_INSTALLER_RETRY = 3
-CURRENT_UBUNTU_VERSION = "18.04"  # full title is ubuntu-18.04
+CURRENT_UBUNTU_VERSION = "20.04"  # full title is ubuntu-20.04
 MAX_PRECACHED_VERSION = "3.9.0"
 
 UNIX_INSTALL_ARRAY = ["sh", "setup.sh"]

scripts/devops_tasks/setup_execute_tests.py

@@ -25,6 +25,7 @@
     clean_coverage,
     is_error_code_5_allowed,
     create_code_coverage_params,
+    filter_packages_by_compatibility_override
 )
 from tox_harness import prep_and_run_tox
 
@@ -295,15 +296,20 @@ def execute_global_install_and_test(
         target_dir = root_dir
 
     targeted_packages = process_glob_string(args.glob_string, target_dir, "", args.filter_type)
+    compatible_targeted_packages = filter_packages_by_compatibility_override(targeted_packages)
+
+    if targeted_packages != compatible_targeted_packages:
+        logging.info("At least one package incompatible with current platform was detected. Skipping: {}".format(set(targeted_packages) - set(compatible_targeted_packages)))
+
     extended_pytest_args = []
 
-    if len(targeted_packages) == 0:
+    if len(compatible_targeted_packages) == 0:
         exit(0)
 
     if args.xdist:
         extended_pytest_args.extend(["-n", "8", "--dist=loadscope"])
 
     if args.runtype != "none":
-        execute_global_install_and_test(args, targeted_packages, extended_pytest_args)
+        execute_global_install_and_test(args, compatible_targeted_packages, extended_pytest_args)
     else:
-        prep_and_run_tox(targeted_packages, args, extended_pytest_args)
+        prep_and_run_tox(compatible_targeted_packages, args, extended_pytest_args)

sdk/core/azure-core-tracing-opentelemetry/tests/conftest.py

@@ -4,6 +4,7 @@
 # ------------------------------------
 from opentelemetry import trace
 from opentelemetry.sdk.trace import TracerProvider
+import sys
 
 import pytest
 

sdk/core/azure-core/tests/test_basic_transport.py

@@ -1200,6 +1200,6 @@ def test_conflict_timeout(caplog, port, http_request):
 def test_aiohttp_loop():
     import asyncio
     from azure.core.pipeline.transport import AioHttpTransport
-    loop = asyncio._get_running_loop()
+    loop = asyncio.get_event_loop()
     with pytest.raises(ValueError):
         transport = AioHttpTransport(loop=loop)