merge latest

Author: robga

.sdkauto/generateScript.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+PARAMS=$(cat $1 | jq '{localPath: .specFolder, readmeFiles: .relatedReadmeMdFiles, $outputPath}' -c --arg outputPath $2)
+
+echo $PARAMS
+
+pushd generator
+
+npm run generate-all $PARAMS

.sdkauto/initScript.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+npm install -g npm
+
+pushd generator
+
+npm install
+
+popd

.vscode/launch.json

@@ -34,7 +34,20 @@
       "name": "Generate all",
       "runtimeArgs": ["-r", "ts-node/register"],
       "cwd": "${workspaceFolder}/generator",
-      "args": ["${workspaceFolder}/generator/cmd/generateall.ts"]
+      "args": [
+        "${workspaceFolder}/generator/cmd/generateall.ts"
+      ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Generate all - local",
+      "runtimeArgs": ["-r", "ts-node/register"],
+      "cwd": "${workspaceFolder}/generator",
+      "args": [
+        "${workspaceFolder}/generator/cmd/generateall.ts",
+        "{\"localPath\":\"../azure-rest-api-specs\",\"readmeFiles\":[\"specification/signalr/resource-manager/readme.md\",\"specification/appplatform/resource-manager/readme.md\"],\"outputPath\":\"../generateOutput.json\"}"
+      ]
     }
   ],
   "inputs": [

azure-pipelines-generate-onboarded-report.yml

@@ -0,0 +1,69 @@
+pr: none
+trigger: none
+
+schedules:
+- cron: "0 6 * * 5"
+  branches:
+    include:
+    - master
+  displayName: Weekly report of onboarded RPs
+
+jobs:
+- job: Generate
+  timeoutInMinutes: 180
+
+  pool:
+    vmImage: 'ubuntu-latest'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - task: DownloadSecureFile@1
+    name: schemasDeployKey
+    inputs:
+      secureFile: schemas_rsa
+    displayName: "Download GitHub Deploy Key"
+  
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '14.x'
+
+  - script: |
+      set -Eeuxo pipefail
+      mkdir ~/.ssh && mv $(schemasDeployKey.secureFilePath) ~/.ssh/schemas_rsa
+      chmod 700 ~/.ssh
+      chmod 600 ~/.ssh/schemas_rsa
+      git config user.name "Autogenerator Pipeline"
+      git config user.email "azure-resource-manager-schemas@noreply.github.com"
+      git config core.sshCommand "ssh -i ~/.ssh/schemas_rsa -F /dev/null"
+      git remote set-url origin git@github.com:Azure/azure-resource-manager-schemas.git
+      ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+      git fetch
+    displayName: Configure git
+
+  - script: |
+      set -Eeuxo pipefail
+      git checkout $(Build.SourceBranchName)
+    displayName: Checkout repo
+
+  - script: |
+      set -Eeuxo pipefail
+      cd generator
+      npm install
+    displayName: 'Install packages'
+
+  - script: |
+      set -Eeuxo pipefail
+      cd generator
+      npm run generate-onboarded-report
+    displayName: 'Generate report'
+
+  - script: |
+      set -Eeuxo pipefail
+      git add --all onboarded-report
+      if ! git diff-index --quiet HEAD --; then
+        git commit -m "Generated report"
+        git push origin $(Build.SourceBranchName)
+      fi
+    displayName: Commit report

generator/README.md

@@ -34,9 +34,8 @@ View the Pipeline Configuration [here](/azure-pipelines-autogen.yml).
 #### Onboarding teams to autogeneration pipeline
 
 1. Run `npm run list-basepaths` to discover the specs repo path containing the swagger definitions.
-2. Run `npm run generate-single {basePath}` to generate schemas for them. Save the console output to file.
-3. Remove all references to the provider namespace from [schemas/2019-04-01/deploymentTemplate.json](/schemas/2019-04-01/deploymentTemplate.json).
-4. If there are any modifications to [schemas/common/autogeneratedResources.json](/schemas/common/autogeneratedResources.json), you may need to make corresponding changes to [schemas/2014-04-01-preview/deploymentTemplate.json](/schemas/2014-04-01-preview/deploymentTemplate.json) and [schemas/2015-01-01/deploymentTemplate.json](/schemas/2015-01-01/deploymentTemplate.json).
-5. Add the {basePath} to the autogenlist array in [autogenlist.ts](./autogenlist.ts).
-6. Commit all the changed files, and submit a PR to this repo with title "Onboarding {provider} for autogeneration". Include the full output saved in step 2 as a comment.
-7. Ensure all CI tests pass, and ask the team to review the PR (including tool output) to ensure that all expected resources have been detected and generated as expected.
+1. Run `npm run generate-single {basePath}` to generate schemas for them. Save the console output to file.
+1. Remove all references to the provider namespace from [schemas/2019-04-01/deploymentTemplate.json](/schemas/2019-04-01/deploymentTemplate.json).
+1. Add the {basePath} to the autogenlist array in [autogenlist.ts](./autogenlist.ts).
+1. Commit all the changed files, and submit a PR to this repo with title "Onboarding {provider} for autogeneration". Include the full output saved in step 2 as a comment.
+1. Ensure all CI tests pass, and ask the team to review the PR (including tool output) to ensure that all expected resources have been detected and generated as expected.

generator/autogenlist.ts

@@ -1,11 +1,13 @@
-import { ScopeType, AutogenlistConfig } from './models';
+import { ScopeType, AutoGenConfig } from './models';
+import { postProcessor as insightsApplicationPostProcessor } from './processors/Microsoft.Insights.Application';
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
 import { postProcessor as machineLearningPostProcessor } from './processors/Microsoft.MachineLearning';
+import { postProcessor as machineLearningServicesPostProcessor } from './processors/Microsoft.MachineLearningServices';
 import { postProcessor as storageProcessor } from './processors/Microsoft.Storage';
 import { lowerCaseEquals } from './utils';
 
 // Run "npm run list-basepaths" to discover all the valid readme files to add to this list
-const autogenlist: AutogenlistConfig[] = [
+const autoGenList: AutoGenConfig[] = [
     {
         basePath: 'addons/resource-manager',
         namespace: 'Microsoft.Addons',
@@ -264,10 +266,10 @@ const autogenlist: AutogenlistConfig[] = [
         basePath: 'EnterpriseKnowledgeGraph/resource-manager',
         namespace: 'Microsoft.EnterpriseKnowledgeGraph',
     },
-    { 
+    /*{ NOTE(jcotillo): Temporally removing this RP - latest swagger contains an unsupported type by the schema generator tool
         basePath: 'domainservices/resource-manager',
         namespace: 'Microsoft.AAD',
-    },
+    },*/
     {
         basePath: 'eventhub/resource-manager',
         namespace: 'Microsoft.EventHub',
@@ -332,6 +334,11 @@ const autogenlist: AutogenlistConfig[] = [
         namespace: 'Microsoft.MachineLearning',
         postProcessor: machineLearningPostProcessor,
     },
+    {
+        basePath: 'machinelearningservices/resource-manager',
+        namespace: 'Microsoft.MachineLearningServices',
+        postProcessor: machineLearningServicesPostProcessor,
+    },
     {
         basePath: 'machinelearningcompute/resource-manager',
         namespace: 'Microsoft.MachineLearningCompute',
@@ -378,10 +385,6 @@ const autogenlist: AutogenlistConfig[] = [
         basePath: 'marketplace/resource-manager',
         namespace: 'Microsoft.Marketplace',
     },
-    {
-        basePath: 'machinelearningservices/resource-manager',
-        namespace: 'Microsoft.MachineLearningServices',
-    },
     {
         basePath: 'mysql/resource-manager',
         namespace: 'Microsoft.DBforMySQL',
@@ -697,6 +700,23 @@ const autogenlist: AutogenlistConfig[] = [
     {
         basePath: 'keyvault/resource-manager',
         namespace: 'Microsoft.KeyVault',
+        readmeTag: {
+            '2016-10-01': [
+                'Microsoft.KeyVault/stable/2016-10-01/keyvault.json',
+                'Microsoft.KeyVault/stable/2016-10-01/providers.json',
+                'Microsoft.KeyVault/stable/2016-10-01/secrets.json',
+            ],
+            '2018-02-14': [
+                'Microsoft.KeyVault/stable/2018-02-14/keyvault.json',
+                'Microsoft.KeyVault/stable/2018-02-14/providers.json',
+                'Microsoft.KeyVault/stable/2018-02-14/secrets.json',
+            ],
+            '2018-02-14-preview': [
+                'Microsoft.KeyVault/preview/2018-02-14-preview/keyvault.json',
+                'Microsoft.KeyVault/preview/2018-02-14-preview/providers.json',
+                'Microsoft.KeyVault/preview/2018-02-14-preview/secrets.json',
+            ],
+        }
     },
     {
         basePath: 'hybridkubernetes/resource-manager',
@@ -732,6 +752,7 @@ const autogenlist: AutogenlistConfig[] = [
         basePath: 'applicationinsights/resource-manager',
         namespace: 'Microsoft.Insights',
         suffix: 'Application',
+        postProcessor: insightsApplicationPostProcessor,
     },
     {
         basePath: 'quantum/resource-manager',
@@ -749,10 +770,10 @@ const autogenlist: AutogenlistConfig[] = [
     }
 ];
 
-export function getAutogenlist(): AutogenlistConfig[] {
-    return autogenlist;
+export function getAutoGenList(): AutoGenConfig[] {
+    return autoGenList;
 }
 
-export function findAutogenEntries(basePath: string): AutogenlistConfig[] {
-    return autogenlist.filter(w => lowerCaseEquals(w.basePath, basePath));
+export function findAutogenEntries(basePath: string): AutoGenConfig[] {
+    return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath));
 }

generator/cmd/findbasepath.ts

@@ -15,9 +15,9 @@ executeSynchronous(async () => {
     // format basePath.
     basePath = `${basePath}/resource-manager`;
 
-    const autogenEntries = findAutogenEntries(basePath);
+    const autoGenEntries = findAutogenEntries(basePath);
 
-    if (autogenEntries.length === 0) {
+    if (autoGenEntries.length === 0) {
         // not onboarded in autogeneration
         console.log("false");
         return;

generator/cmd/generateall.ts

@@ -1,14 +1,18 @@
 import * as constants from '../constants';
-import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
-import { SchemaConfiguration, generateSchemas, clearAutogeneratedSchemaRefs, saveAutogeneratedSchemaRefs } from '../generate';
-import { getAutogenlist } from '../autogenlist';
+import { cloneAndGenerateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
+import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs } from '../generate';
+import { getAutoGenList } from '../autogenlist';
 import chalk from 'chalk';
-import { flatten, chunk } from 'lodash';
-import { executeSynchronous, chunker } from '../utils';
+import { flatten } from 'lodash';
+import { executeSynchronous, chunker, writeJsonFile } from '../utils';
+import { Package } from '../models';
 
 interface GenerateAllParams {
     batchCount?: number,
     batchIndex?: number,
+    localPath?: string,
+    readmeFiles?: string[],
+    outputPath?: string,
 }
 
 function parseParams(): GenerateAllParams {
@@ -22,34 +26,67 @@ function parseParams(): GenerateAllParams {
 executeSynchronous(async () => {
     const params = parseParams();
 
-    let filteredAutogenlist = getAutogenlist();
+    let filteredAutoGenList = getAutoGenList();
     if (params.batchCount !== undefined && params.batchIndex !== undefined) {
-        filteredAutogenlist = chunker(filteredAutogenlist, params.batchCount)[params.batchIndex];
+        filteredAutoGenList = chunker(filteredAutoGenList, params.batchCount)[params.batchIndex];
     }
 
-    await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    let localPath = params.localPath;
+    if (!localPath) {
+        localPath = constants.specsRepoPath;
+        await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    } else {
+        localPath = await resolveAbsolutePath(localPath);
+    }
+
+    if (!!params.readmeFiles) {
+        filteredAutoGenList = filteredAutoGenList.filter(c => {
+            let r = params.readmeFiles?.find(f => f.startsWith('specification/' + c.basePath));
+            if (!!r) {
+                c.readmeFile = r;
+                return true;
+            }
+            return false;
+        }
+        );
+    }
 
-    await clearAutogeneratedSchemaRefs(filteredAutogenlist);
+    await clearAutoGeneratedSchemaRefs(filteredAutoGenList);
 
     const schemaConfigs: SchemaConfiguration[] = [];
     const errors = [];
-    for (const autogenlistConfig of filteredAutogenlist) {
+    const packages: Package[] = [];
+    for (const autoGenConfig of filteredAutoGenList) {
+        let pkg = {
+            path: ['schemas']
+        } as Package;
         try {
-            const readme = await validateAndReturnReadmePath(autogenlistConfig.basePath);
+            const readme = await validateAndReturnReadmePath(localPath, autoGenConfig.readmeFile || autoGenConfig.basePath);
+            pkg.packageName = getPackageString(readme);
 
-            const newConfigs = await generateSchemas(readme, autogenlistConfig);
+            const newConfigs = await generateSchemas(readme, autoGenConfig);
             schemaConfigs.push(...newConfigs);
+            pkg.result = 'succeeded';
         } catch(error) {
-            console.log(chalk.red(`Caught exception processing autogenlist entry ${autogenlistConfig.basePath}.`));
+            pkg.packageName = autoGenConfig.basePath;
+            pkg.result = 'failed';
+            console.log(chalk.red(`Caught exception processing autogenlist entry ${autoGenConfig.basePath}.`));
             console.log(chalk.red(error));
 
             errors.push(error);
         }
+        packages.push(pkg);
     }
 
-    await saveAutogeneratedSchemaRefs(flatten(schemaConfigs));
+    await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
 
-    if (errors.length > 0) {
-        throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
+    if (!!params.outputPath) {
+        const outputPath = await resolveAbsolutePath(params.outputPath);
+        await writeJsonFile(outputPath, { packages });
+    } else {
+        if (errors.length > 0) {
+            throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
+        }
     }
+
 });