Enable Build of Non-Universal Wheels (Azure#32911)

* Temporarily Scaffold new azure-storage-extensions package
* Update parse_functions.py to pick up extension modules and package names
* Add additional code testing parse_function for the new functionality
* Update build.py to invoke cibuildwheel to produce platform-specific wheels where necessary

Co-authored-by: Jacob Lauzon <jalauzon@microsoft.com>
Co-authored-by: Azure SDK Bot <adxpysdk@microsoft.com>

Author: 3 people

.vscode/cspell.json

@@ -127,6 +127,7 @@
     "conda/conda-releaselogs/azure-mgmt.md"
   ],
   "words": [
+    "cibuildwheel",
     "aoai",
     "pyprojects",
     "certifi",
@@ -650,7 +651,7 @@
         "engelska",
         "fhir",
         "FHIR",
-        "naam",
+        "naam"
       ]
     },
     {

eng/ci_tools.txt

@@ -13,6 +13,7 @@ wrapt==1.12.1; python_version <= '3.10'
 wrapt==1.15.0; python_version >= '3.11'
 typing-extensions<=4.6.3
 pyproject-api<1.6
+cibuildwheel==2.16.2 ; python_version > '3.8'
 
 # requirements leveraged for testing
 pytest==7.3.1

eng/pipelines/templates/jobs/ci.yml

@@ -61,31 +61,105 @@ parameters:
   - name: GenerateApiReviewForManualOnly
     type: boolean
     default: false
+  - name: AdvancedBuild
+    type: boolean
+    default: false
 
 jobs:
-  - job: 'Build'
-    timeoutInMinutes: 90
+  
+  - ${{ if eq(parameters['AdvancedBuild'], false) }}:
+    - job: 'Build'
+      timeoutInMinutes: 90
 
-    pool:
-      name: azsdk-pool-mms-ubuntu-2004-general
-      vmImage: MMSUbuntu20.04
+      pool:
+        name: azsdk-pool-mms-ubuntu-2004-general
+        vmImage: MMSUbuntu20.04
 
-    steps:
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
-      parameters:
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+      steps:
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+        parameters:
+          BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
-    - template: ../steps/build-package-artifacts.yml
-      parameters:
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
-        TestPipeline: ${{ parameters.TestPipeline }}
-        Artifacts: ${{ parameters.Artifacts }}
+      - template: ../steps/build-package-artifacts.yml
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+          TestPipeline: ${{ parameters.TestPipeline }}
+          Artifacts: ${{ parameters.Artifacts }}
+
+  - ${{ if eq(parameters['AdvancedBuild'], true) }}:
+    - job: 'Build'
+      timeoutInMinutes: 90
+
+      strategy:
+        matrix:
+          Linux:
+            imageName: 'MMSUbuntu22.04'
+            poolName: 'azsdk-pool-mms-ubuntu-2204-general'
+            ArtifactName: 'linux'
+          Windows:
+            imageName: 'MMS2022'
+            poolName: 'azsdk-pool-mms-win-2022-general'
+            ArtifactName: 'windows'
+          Mac:
+            imageName: 'macos-11'
+            poolName: 'Azure Pipelines'
+            ArtifactName: 'mac'
+
+      pool:
+        name: $(poolName)
+        vmImage: $(imageName)
+
+      steps:
+      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+        parameters:
+          BuildTargetingString: ${{ parameters.BuildTargetingString }}
+
+      - template: ../steps/build-package-artifacts.yml
+        parameters:
+          ServiceDirectory: ${{ parameters.ServiceDirectory }}
+          BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+          TestPipeline: ${{ parameters.TestPipeline }}
+          Artifacts: ${{ parameters.Artifacts }}
+          ArtifactSuffix: $(ArtifactName)
+
+    - job: 'CoalesceBuildArtifacts'
+      displayName: Combine Built Artifacts
+      dependsOn: 
+        - 'Build'
+      timeoutInMinutes: 90
+
+      pool:
+        name: azsdk-pool-mms-ubuntu-2004-general
+        vmImage: MMSUbuntu20.04
+
+      steps:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifactName: 'packages_windows'
+          targetPath: $(Build.ArtifactStagingDirectory)/packages
+
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifactName: 'packages_mac'
+          targetPath: $(Build.ArtifactStagingDirectory)/packages
+
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifactName: 'packages_linux'
+          targetPath: $(Build.ArtifactStagingDirectory)/packages
+
+      - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+        parameters:
+          ArtifactPath: '$(Build.ArtifactStagingDirectory)/packages'
+          ArtifactName: 'packages'
 
   - job: 'Build_Extended'
     displayName: Build Extended
     dependsOn: 
       - 'Build'
+      - ${{ if eq(parameters['AdvancedBuild'], true) }}:
+        - 'CoalesceBuildArtifacts'
     timeoutInMinutes: 90
 
     pool:
@@ -173,6 +247,8 @@ jobs:
       JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
       DependsOn:
         - 'Build'
+        - ${{ if eq(parameters['AdvancedBuild'], true) }}:
+          - 'CoalesceBuildArtifacts'
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
@@ -220,6 +296,8 @@ jobs:
         Cloud: Public
       DependsOn:
         - 'Build'
+        - ${{ if eq(parameters['AdvancedBuild'], true) }}:
+          - 'CoalesceBuildArtifacts'
       AdditionalParameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         ServiceDirectory: ${{ parameters.ServiceDirectory }}

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -72,6 +72,9 @@ parameters:
 - name: GenerateApiReviewForManualOnly
   type: boolean
   default: false
+- name: AdvancedBuild
+  type: boolean
+  default: false
 
 variables:
   - template: /eng/pipelines/templates/variables/globals.yml
@@ -106,6 +109,7 @@ stages:
         ValidateFormatting: ${{ parameters.ValidateFormatting }}
         TestProxy: ${{ parameters.TestProxy }}
         GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
+        AdvancedBuild: ${{ parameters.AdvancedBuild }}
 
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
   - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -14,12 +14,22 @@ parameters:
   - name: DevFeedName
     type: string
     default: 'public/azure-sdk-for-python'
+  - name: ArtifactSuffix
+    type: string
+    default: ''
+
 # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
 #     eng/pipelines/templates/jobs/ci.tests.yml
 #     eng/pipelines/templates/jobs/ci.yml
 #     eng/pipelines/templates/jobs/live.test.yml
 
 # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
+
+# The variable '$(ArtifactName)' is set at runtime by the strategy for the `Build` job.
+# Specifically, one of 3 values `linux`, `mac`, or `windows` will always be populated when invoked from `jobs/ci.yml`.
+
+# However, please note that this variable will not ALWAYS be set. If using `build-artifacts.yml`, ensure that this variable is set AT LEAST
+# to "linux", otherwise the primary tasks will not be run as expected. APIStub, Docs, etc are NOT run on machines that aren't linux.
 steps:
   - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
     parameters:
@@ -71,31 +81,50 @@ steps:
       twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
     displayName: 'Verify Readme'
 
-  - ${{if eq(variables['System.TeamProject'], 'internal') }}:
-    - template: auth-dev-feed.yml
+  - ${{ parameters.BeforePublishSteps }}
+
+  - ${{ if eq(parameters.ArtifactSuffix, '') }}:
+    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
       parameters:
-        DevFeedName: ${{ parameters.DevFeedName }}
+        ArtifactPath: '$(Build.ArtifactStagingDirectory)'
+        ArtifactName: 'packages'
 
-  - ${{ parameters.BeforePublishSteps }}
+    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+      parameters:
+        ArtifactPath: '$(Build.SourcesDirectory)/.logs'
+        ArtifactName: 'build_logs'
 
-  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
-    parameters:
-      ArtifactPath: '$(Build.ArtifactStagingDirectory)'
-      ArtifactName: 'packages'
+    - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+        displayName: 'Generate BOM'
+        condition: succeededOrFailed()
+        inputs:
+          BuildDropPath: $(Build.ArtifactStagingDirectory)
 
-  - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
-    parameters:
-      ArtifactPath: '$(Build.SourcesDirectory)/.logs'
-      ArtifactName: 'build_logs'
+      - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+        parameters:
+          ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest'
+          ArtifactName: 'packages_manifest'
 
-  - ${{if eq(variables['System.TeamProject'], 'internal') }}:
-    - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
-      displayName: 'Generate BOM'
-      condition: succeededOrFailed()
-      inputs:
-        BuildDropPath: $(Build.ArtifactStagingDirectory)
+  - ${{ if ne(parameters.ArtifactSuffix, '') }}:
+    - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+      parameters:
+        ArtifactPath: '$(Build.ArtifactStagingDirectory)'
+        ArtifactName: 'packages_${{ parameters.ArtifactSuffix }}'
 
     - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
       parameters:
-        ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest'
-        ArtifactName: 'packages_manifest'
+        ArtifactPath: '$(Build.SourcesDirectory)/.logs'
+        ArtifactName: 'build_logs_${{ parameters.ArtifactSuffix }}'
+
+    - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+        displayName: 'Generate BOM'
+        condition: succeededOrFailed()
+        inputs:
+          BuildDropPath: $(Build.ArtifactStagingDirectory)
+
+      - template: /eng/common/pipelines/templates/steps/publish-artifact.yml
+        parameters:
+          ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest'
+          ArtifactName: 'packages_manifest_${{ parameters.ArtifactSuffix }}'

eng/tox/create_package_and_install.py

@@ -61,6 +61,7 @@ def discover_packages(setuppy_path, args):
             args.target_setup,
             args.package_type,
         )
+
     return packages
 
 

scripts/devops_tasks/common_tasks.py

@@ -26,7 +26,7 @@
 from packaging.version import Version
 from packaging.version import parse
 
-from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, lambda_filter_azure_pkg, str_to_bool
+from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, NO_TESTS_ALLOWED, lambda_filter_azure_pkg, str_to_bool
 from ci_tools.parsing import parse_require, ParsedSetup
 
 DEV_REQ_FILE = "dev_requirements.txt"
@@ -126,6 +126,7 @@ def is_error_code_5_allowed(target_pkg, pkg_name):
             )
         )
         or pkg_name in MANAGEMENT_PACKAGE_IDENTIFIERS
+        or pkg_name in NO_TESTS_ALLOWED
     ):
         return True
     else:

tools/azure-sdk-tools/ci_tools/build.py

@@ -106,7 +106,12 @@ def build() -> None:
         target_dir = repo_root
 
     targeted_packages = discover_targeted_packages(
-        args.glob_string, target_dir, args.package_filter_string, filter_type="Build", compatibility_filter=True, include_inactive=args.inactive
+        args.glob_string,
+        target_dir,
+        args.package_filter_string,
+        filter_type="Build",
+        compatibility_filter=True,
+        include_inactive=args.inactive,
     )
     artifact_directory = get_artifact_directory(args.distribution_directory)
 
@@ -159,17 +164,20 @@ def create_package(
     """
 
     dist = get_artifact_directory(dest_folder)
+    setup_parsed = ParsedSetup.from_path(setup_directory_or_file)
 
-    if not os.path.isdir(setup_directory_or_file):
-        setup_directory_or_file = os.path.dirname(setup_directory_or_file)
+    if setup_parsed.ext_modules:
+        run_logged(
+            [sys.executable, "-m", "cibuildwheel", "--output-dir", dist], prefix="cibuildwheel", cwd=setup_parsed.folder
+        )
 
     if enable_wheel:
         run_logged(
-            [sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_directory_or_file
+            [sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_parsed.folder
         )
     if enable_sdist:
         run_logged(
             [sys.executable, "setup.py", "sdist", "-d", dist],
             prefix="create_sdist",
-            cwd=setup_directory_or_file,
+            cwd=setup_parsed.folder,
         )

tools/azure-sdk-tools/ci_tools/functions.py

@@ -28,6 +28,10 @@
     "azure-ai-anomalydetector",
 ]
 
+NO_TESTS_ALLOWED = [
+]
+
+
 META_PACKAGES = ["azure", "azure-mgmt", "azure-keyvault"]
 
 REGRESSION_EXCLUDED_PACKAGES = [