Verify autorest in pipeline (Azure#16667)

Azure#13724

Author: seankane-msft

eng/.docsettings.yml

@@ -88,24 +88,15 @@ known_content_issues:
   - ['sdk/eventgrid/azure-eventgrid/README.md', '#4554']
   - ['sdk/graphrbac/azure-graphrbac/README.md', '#4554']
   - ['sdk/loganalytics/azure-loganalytics/README.md', '#4554']
-  - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/README.md', '#4554']
-  - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/tests/perf_tests/README.md', '#4554']
   - ['sdk/servicefabric/azure-servicefabric/README.md', '#4554']
   - ['sdk/storage/azure-storage-nspkg/README.md', '#4554']
-  - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']
-  - ['sdk/storage/azure-storage-file-datalake/swagger/README.md', '#4554']
-  - ['sdk/storage/azure-storage-file-share/swagger/README.md', '#4554']
   - ['sdk/storage/azure-storage/README.md', '#4554']
   - ['sdk/storage/azure-storage-nspkg/README.rst', '#4554']
-  - ['sdk/storage/azure-storage-queue/swagger/README.md', '#4554']
   - ['sdk/storage/README.md', '#4554']
   - ['sdk/textanalytics/azure-ai-textanalytics/samples/README.md', '#4554']
-  - ['sdk/monitor/azure-monitor-opentelemetry-exporter/swagger/README.md', '#4554']
   - ['sdk/monitor/azure-monitor-opentelemetry-exporter/README.md', '#4554']
-  - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
-  - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
   - ['sdk/media/azure-media-nspkg/README.md', '#4554']
   - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554']
   - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
@@ -156,6 +147,18 @@ known_content_issues:
   - ['doc/dev/mgmt/README.md', 'dev readme']
   - ['doc/dev/README.md', dev readme']
   - ['doc/README.md', 'dev readme']
+
+  # swagger files
+  - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
+  - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-file-datalake/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-file-share/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-queue/swagger/README.md', '#4554']
+  - ['sdk/monitor/azure-monitor-opentelemetry-exporter/swagger/README.md', '#4554']
+  - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
+  - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
+  - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
 package_indexing_exclusion_list:
   - 'azure-sdk-tools'
   - 'azure-template'

eng/autorest_req.txt

@@ -4,5 +4,6 @@ pytest-cov==2.8.1
 pytest-asyncio==0.12.0; python_version >= '3.5'
 isodate==0.6.0
 msrest==0.6.14
-aiohttp==3.6.2
-wheel==0.34.2
+wheel==0.34.2
+GitPython==3.1.14
+aiohttp==3.6.2; python_version >= '3.6'

eng/pipelines/templates/jobs/ci.yml

@@ -43,6 +43,9 @@ parameters:
   - name: MatrixReplace
     type: object
     default: []
+  - name: VerifyAutorest
+    type: boolean
+    default: false
 
 jobs:
   - job: 'Build'
@@ -100,6 +103,7 @@ jobs:
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
         TestPipeline: ${{ parameters.TestPipeline }}
         Artifacts: ${{ parameters.Artifacts }}
+        VerifyAutorest: ${{ parameters.VerifyAutorest }}
 
   - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
     parameters:

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -57,6 +57,9 @@ parameters:
 - name: MatrixReplace
   type: object
   default: []
+- name: VerifyAutorest
+  type: boolean
+  default: false
 
 stages:
   - stage: Build
@@ -88,6 +91,7 @@ stages:
           - OsVmImage=macOS-10.15/MMSUbuntu18.04
           - ${{ each replacement in parameters.MatrixReplace }}:
             - ${{ replacement }}
+        VerifyAutorest: ${{ parameters.VerifyAutorest }}
 
 
   # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
@@ -115,4 +119,4 @@ stages:
         ArtifactName: conda
         ${{ if eq(parameters.ServiceDirectory, 'template') }}:
           TestPipeline: true
-        
+

eng/pipelines/templates/steps/analyze.yml

@@ -5,6 +5,7 @@ parameters:
   AdditionalTestArgs: ''
   Artifacts: []
   TestPipeline: false
+  VerifyAutorest: false
 
 steps:
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
@@ -57,6 +58,11 @@ steps:
     parameters:
       SourceDirectory: $(Build.SourcesDirectory)
 
+  - template: ../steps/verify-autorest.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      VerifyAutorest: ${{ parameters.VerifyAutorest }}
+
   - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
     # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR
     # builds should be sufficient.

eng/pipelines/templates/steps/verify-autorest.yml

@@ -0,0 +1,66 @@
+parameters:
+  - name: ServiceDirectory
+    type: string
+  - name: NodeVersion
+    type: string
+    default: '12.x'
+  - name: PythonVersion
+    type: string
+    default: '$(PythonVersion)'
+  - name: auto_rest_clone_url
+    type: string
+    default: 'https://github.com/Azure/autorest.python.git'
+  - name: VerifyAutorest
+    type: boolean
+    default: false
+
+steps:
+  - ${{if eq(parameters.VerifyAutorest, 'true')}}:
+    - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+    - task: UsePythonVersion@0
+      displayName: Use
+      inputs:
+        versionSpec: ${{ parameters.PythonVersion }}
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: ${{ parameters.NodeVersion }}
+
+    - script: |
+        npm install -g autorest
+        autorest --help
+      displayName: "Install autorest"
+
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        Paths:
+          - "/*"
+        Repositories:
+          - Name: "Azure/autorest.python"
+            Commitish: "master"
+            WorkingDirectory: "$(Build.SourcesDirectory)/autorest.python"
+        SkipDefaultCheckout: true
+
+    - script: |
+        python --version
+        pip install -r eng/autorest_req.txt
+        cd autorest.python
+        npm install
+      displayName: 'Prepare Environment'
+
+    - task: PythonScript@0
+      displayName: 'Verify autorest'
+      inputs:
+        scriptPath: 'scripts/devops_tasks/verify_autorest.py'
+        arguments: >-
+          --service_directory="${{ parameters.ServiceDirectory }}"
+
+    - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
+      - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+    #   # displayName: Create PR for ${{ ServiceDirectory }}
+    #   condition: and(ne(variables['Build.Reason'],'PullRequest'), eq(variables['System.TeamProject'], 'internal'))
+        parameters:
+          CommitMsg: "Regenerated code from nightly builds"
+          PRTitle: "Automated autorest generation"
+          PRBranchName: 'autorest-${{ ServiceDirectory }}'

scripts/devops_tasks/verify_autorest.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import argparse
+import os
+import logging
+import sys
+
+from common_tasks import run_check_call
+
+logging.getLogger().setLevel(logging.INFO)
+
+root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
+sdk_dir = os.path.join(root_dir, "sdk")
+
+SWAGGER_FOLDER = "swagger"
+
+
+def run_autorest(service_dir):
+    logging.info("Running autorest for {}".format(service_dir))
+
+    service_dir = os.path.join(sdk_dir, service_dir)
+
+    swagger_folders = find_swagger_folders(service_dir)
+
+    for working_dir in swagger_folders:
+        os.chdir(working_dir)
+        f = os.path.abspath(os.path.join(working_dir, "README.md"))
+        if os.path.exists(f):
+            reset_command = ["autorest", "--reset"]
+            run_check_call(reset_command, root_dir)
+
+            command = ["autorest", "--python", f, "--verbose"]
+            logging.info("Command: {}\nLocation: {}\n".format(command, working_dir))
+            run_check_call(command, working_dir)
+    return swagger_folders
+
+
+def find_swagger_folders(directory):
+    logging.info("Searching for swagger files in: {}".format(directory))
+
+    ret = []
+    for root, subdirs, files in os.walk(directory):
+        for d in subdirs:
+            if d == SWAGGER_FOLDER:
+                if os.path.exists(os.path.join(root, d, "README.md")):
+                    ret.append(os.path.join(root, d))
+
+    logging.info("Found swagger files at: {}".format(ret))
+    return ret
+
+
+def check_diff(folder):
+    # We don't care about changes to txt files (dev_requirements change)
+    run_check_call(["git", "status"], sdk_dir, always_exit=False)
+
+    command = [
+        "git",
+        "checkout",
+        "--",
+        "**/*.txt",
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+
+    # Remove the whl dirs
+    command = [
+        "rm",
+        "-r",
+        "**/.tmp_whl_dir/"
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+
+    # Next we need to move the autorest and _tox_logs directories and then replace them
+
+    dir_changed = folder.split("/")[:-2]
+    command = [
+        "git",
+        "diff",
+        "--exit-code",
+        "{}".format("/".join(dir_changed)),
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+    if result:
+        command = ["git", "status"]
+        run_check_call(command, root_dir)
+        raise ValueError(
+            "Found difference between re-generated code and current commit. Please re-generate with the latest autorest."
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Run autorest to verify generated code."
+    )
+    parser.add_argument(
+        "--service_directory", help="Directory of the package being tested"
+    )
+
+    args = parser.parse_args()
+    folders = run_autorest(args.service_directory)
+
+    if len(folders):
+        for folder in folders:
+            check_diff(folder)

sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/_configuration.py

@@ -51,7 +51,7 @@ def __init__(
         self.endpoint = endpoint
         self.sync_token = sync_token
         self.api_version = "1.0"
-        self.credential_scopes = kwargs.pop('credential_scopes', [])
+        self.credential_scopes = kwargs.pop('credential_scopes', ['https://dev.azuresynapse.net/.default'])
         kwargs.setdefault('sdk_moniker', 'appconfiguration/{}'.format(VERSION))
         self._configure(**kwargs)
 
@@ -69,7 +69,5 @@ def _configure(
         self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs)
         self.redirect_policy = kwargs.get('redirect_policy') or policies.RedirectPolicy(**kwargs)
         self.authentication_policy = kwargs.get('authentication_policy')
-        if not self.credential_scopes and not self.authentication_policy:
-            raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs")
         if self.credential and not self.authentication_policy:
             self.authentication_policy = policies.BearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs)

sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/aio/_configuration.py

@@ -48,7 +48,7 @@ def __init__(
         self.endpoint = endpoint
         self.sync_token = sync_token
         self.api_version = "1.0"
-        self.credential_scopes = kwargs.pop('credential_scopes', [])
+        self.credential_scopes = kwargs.pop('credential_scopes', ['https://dev.azuresynapse.net/.default'])
         kwargs.setdefault('sdk_moniker', 'appconfiguration/{}'.format(VERSION))
         self._configure(**kwargs)
 
@@ -65,7 +65,5 @@ def _configure(
         self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs)
         self.redirect_policy = kwargs.get('redirect_policy') or policies.AsyncRedirectPolicy(**kwargs)
         self.authentication_policy = kwargs.get('authentication_policy')
-        if not self.credential_scopes and not self.authentication_policy:
-            raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs")
         if self.credential and not self.authentication_policy:
             self.authentication_policy = policies.AsyncBearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs)

sdk/appconfiguration/azure-appconfiguration/swagger/README.md

@@ -15,4 +15,4 @@ enable-xml: true
 vanilla: true
 add-credentials: true
 credential-scopes: https://dev.azuresynapse.net/.default
-```
+```