Target 3.11-rc.1 in nightly python build (Azure#25726)

* target 3.11
* ensure that proxy tests will work in the nightly pipeline
* disable aiohttp extensions to allow installation on py311
* pass filter type of "None" to avoid compatibility filters stopping tests from running on the py311 invocations

Author: scbedd

eng/pipelines/templates/jobs/tests-nightly-python.yml

@@ -7,12 +7,12 @@ jobs:
     displayName: Validate Release Candidate Python - Windows
     variables:
       skipComponentGovernanceDetection: true
-      PythonVersion: '3.10.0-rc.2'
+      PythonVersion: '3.11.0-rc.1'
 
     timeoutInMinutes: 90
 
     pool:
-      vmImage: 'windows-2019'
+      name: 'azsdk-pool-mms-win-2019-general'
 
     steps:
       - template: /eng/pipelines/templates/steps/use-python-version.yml
@@ -22,14 +22,21 @@ jobs:
       - script: |
           python -m pip freeze
           python -m pip --version
-          pip install setuptools==56.0.0 wheel==0.37.0 tox==3.24.3 tox-monorepo==0.1.2 packaging==21.0
+          pip install setuptools==56.0.0 wheel==0.37.0 tox==3.24.3 tox-monorepo==0.1.2 packaging==21.0 requests
+          pip install $(Build.SourcesDirectory)/tools/azure-sdk-tools[build]
         displayName: Install Dependencies
 
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+        parameters:
+          runProxy: false
+
       - script: |
-          python ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl"
+          python ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="None"
         displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
         env:
           YARL_NO_EXTENSIONS: 1
+          AIOHTTP_NO_EXTENSIONS: 1
+          PROXY_URL: "http://localhost:5000"
         continueOnError: true
 
       - task: PublishTestResults@2
@@ -43,32 +50,40 @@ jobs:
     displayName: Validate Release Candidate Python - Linux
     variables:
       skipComponentGovernanceDetection: true
-      PythonVersion: '3.10.0-rc.2'
+      PythonVersion: '3.11.0-rc.1'
 
     timeoutInMinutes: 90
 
     pool:
-      vmImage: 'ubuntu-18.04'
+      name: 'azsdk-pool-mms-ubuntu-2004-general'
 
     steps:
       - template: /eng/pipelines/templates/steps/use-python-version.yml
         parameters:
           versionSpec: $(PythonVersion)
 
       - script: |
+          sudo apt-get update
+          sudo apt-get install build-essential -y
           python -m pip freeze
           python -m pip --version
-          pip install setuptools==56.0.0 wheel==0.37.0 tox==3.24.3 tox-monorepo==0.1.2 packaging==21.0
+          pip install setuptools==56.0.0 wheel==0.37.0 tox==3.24.3 tox-monorepo==0.1.2 packaging==21.0 requests
+          pip install $(Build.SourcesDirectory)/tools/azure-sdk-tools[build]
         displayName: Install Dependencies
 
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+        parameters:
+          runProxy: false
+
       - script: |
           find /usr/lib -name "libffi.so*"
           ln -s /usr/lib/x86_64-linux-gnu/libffi.so.7 /usr/lib/x86_64-linux-gnu/libffi.so.6
-
-          python ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl"
+          python ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="None"
         displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
         env:
           YARL_NO_EXTENSIONS: 1
+          AIOHTTP_NO_EXTENSIONS: 1
+          PROXY_URL: "http://localhost:5000"
         continueOnError: true
 
       - task: PublishTestResults@2
@@ -86,13 +101,18 @@ jobs:
     timeoutInMinutes: 90
 
     pool:
-      vmImage: 'ubuntu-20.04'
+      name: 'azsdk-pool-mms-ubuntu-2004-general'
 
     steps:
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.7 For Build Tools'
+        displayName: 'Use Python 3.9 For Build Tools'
         inputs:
-          versionSpec: '3.7'
+          versionSpec: '3.9'
+
+
+      - template: /eng/common/testproxy/test-proxy-tool.yml
+        parameters:
+          runProxy: false
 
       - script: |
           sudo apt-get update
@@ -114,10 +134,12 @@ jobs:
           python3 -m pip install tox tox-monorepo packaging twine codecov beautifulsoup4
           python3 --version
           cd $(Build.SourcesDirectory)
-          python3 ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl"
+          python3 ./scripts/devops_tasks/setup_execute_tests.py "$(BuildTargetingString)" --junitxml="junit/test_results_38.xml" --toxenv="whl" --filter-type="None"
         displayName: 'Setup - Run Filtered Tests "Nightly" using Python Edge'
         env:
           YARL_NO_EXTENSIONS: 1
+          PROXY_URL: "http://localhost:5000"
+          AIOHTTP_NO_EXTENSIONS: 1
         continueOnError: true
 
       - task: PublishTestResults@2

eng/scripts/Language-Settings.ps1

@@ -574,7 +574,7 @@ function Import-Dev-Cert-python
   Write-Host "Python Trust Methodology"
 
   $pathToScript = Resolve-Path (Join-Path -Path $PSScriptRoot -ChildPath "../../scripts/devops_tasks/trust_proxy_cert.py")
-
+  python -m pip install requests
   python $pathToScript
 }
 

tools/azure-sdk-tools/ci_tools/functions.py

@@ -144,12 +144,14 @@ def discover_targeted_packages(
 
     # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*"
     collected_directories = list(set([p for p in collected_top_level_directories if additional_contains_filter in p]))
+    pkg_set_ci_filtered = collected_directories
 
     # if we have individually queued this specific package, it's obvious that we want to build it specifically
     # in this case, do not honor the omission list
     if len(collected_directories) == 1:
         if compatibility_filter:
             pkg_set_ci_filtered = filter_for_compatibility(collected_directories)
+
     # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted
     # packages
     else: