Merge pull request #1582 from Azure/sdkauto

Sdkauto

Author: leniatgh

.sdkauto/initScript.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+npm install -g npm
+
 pushd generator
 
 npm install

.vscode/launch.json

@@ -46,7 +46,7 @@
       "cwd": "${workspaceFolder}/generator",
       "args": [
         "${workspaceFolder}/generator/cmd/generateall.ts",
-        "{\"localPath\":\"/home/me/workspace/azure-rest-api-specs\",\"readmeFiles\":[\"specification/signalr/resource-manager/readme.md\",\"specification/appplatform/resource-manager/readme.md\"],\"outputPath\":\"../generateOutput.json\"}"
+        "{\"localPath\":\"../azure-rest-api-specs\",\"readmeFiles\":[\"specification/signalr/resource-manager/readme.md\",\"specification/appplatform/resource-manager/readme.md\"],\"outputPath\":\"../generateOutput.json\"}"
       ]
     }
   ],

generator/autogenlist.ts

@@ -1,4 +1,4 @@
-import { ScopeType, AutogenlistConfig } from './models';
+import { ScopeType, AutoGenConfig } from './models';
 import { postProcessor as insightsApplicationPostProcessor } from './processors/Microsoft.Insights.Application';
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
 import { postProcessor as machineLearningPostProcessor } from './processors/Microsoft.MachineLearning';
@@ -7,7 +7,7 @@ import { postProcessor as storageProcessor } from './processors/Microsoft.Storag
 import { lowerCaseEquals } from './utils';
 
 // Run "npm run list-basepaths" to discover all the valid readme files to add to this list
-const autogenlist: AutogenlistConfig[] = [
+const autoGenList: AutoGenConfig[] = [
     {
         basePath: 'addons/resource-manager',
         namespace: 'Microsoft.Addons',
@@ -674,6 +674,23 @@ const autogenlist: AutogenlistConfig[] = [
     {
         basePath: 'keyvault/resource-manager',
         namespace: 'Microsoft.KeyVault',
+        readmeTag: {
+            '2016-10-01': [
+                'Microsoft.KeyVault/stable/2016-10-01/keyvault.json',
+                'Microsoft.KeyVault/stable/2016-10-01/providers.json',
+                'Microsoft.KeyVault/stable/2016-10-01/secrets.json',
+            ],
+            '2018-02-14': [
+                'Microsoft.KeyVault/stable/2018-02-14/keyvault.json',
+                'Microsoft.KeyVault/stable/2018-02-14/providers.json',
+                'Microsoft.KeyVault/stable/2018-02-14/secrets.json',
+            ],
+            '2018-02-14-preview': [
+                'Microsoft.KeyVault/stable/2018-02-14-preview/keyvault.json',
+                'Microsoft.KeyVault/stable/2018-02-14-preview/providers.json',
+                'Microsoft.KeyVault/stable/2018-02-14-preview/secrets.json',
+            ],
+        }
     },
     {
         basePath: 'hybridkubernetes/resource-manager',
@@ -727,10 +744,10 @@ const autogenlist: AutogenlistConfig[] = [
     }
 ];
 
-export function getAutogenlist(): AutogenlistConfig[] {
-    return autogenlist;
+export function getAutoGenList(): AutoGenConfig[] {
+    return autoGenList;
 }
 
-export function findAutogenEntries(basePath: string): AutogenlistConfig[] {
-    return autogenlist.filter(w => lowerCaseEquals(w.basePath, basePath));
+export function findAutogenEntries(basePath: string): AutoGenConfig[] {
+    return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath));
 }

generator/cmd/findbasepath.ts

@@ -15,9 +15,9 @@ executeSynchronous(async () => {
     // format basePath.
     basePath = `${basePath}/resource-manager`;
 
-    const autogenEntries = findAutogenEntries(basePath);
+    const autoGenEntries = findAutogenEntries(basePath);
 
-    if (autogenEntries.length === 0) {
+    if (autoGenEntries.length === 0) {
         // not onboarded in autogeneration
         console.log("false");
         return;

generator/cmd/generateall.ts

@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
-import { SchemaConfiguration, generateSchemas, clearAutogeneratedSchemaRefs, saveAutogeneratedSchemaRefs } from '../generate';
-import { getAutogenlist } from '../autogenlist';
+import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs } from '../generate';
+import { getAutoGenList } from '../autogenlist';
 import chalk from 'chalk';
 import { flatten } from 'lodash';
 import { executeSynchronous, chunker, writeJsonFile } from '../utils';
@@ -26,9 +26,9 @@ function parseParams(): GenerateAllParams {
 executeSynchronous(async () => {
     const params = parseParams();
 
-    let filteredAutogenlist = getAutogenlist();
+    let filteredAutoGenList = getAutoGenList();
     if (params.batchCount !== undefined && params.batchIndex !== undefined) {
-        filteredAutogenlist = chunker(filteredAutogenlist, params.batchCount)[params.batchIndex];
+        filteredAutoGenList = chunker(filteredAutoGenList, params.batchCount)[params.batchIndex];
     }
 
     let localPath = params.localPath;
@@ -40,7 +40,7 @@ executeSynchronous(async () => {
     }
 
     if (!!params.readmeFiles) {
-        filteredAutogenlist = filteredAutogenlist.filter(c => {
+        filteredAutoGenList = filteredAutoGenList.filter(c => {
             let r = params.readmeFiles?.find(f => f.startsWith('specification/' + c.basePath));
             if (!!r) {
                 c.readmeFile = r;
@@ -51,34 +51,34 @@ executeSynchronous(async () => {
         );
     }
 
-    await clearAutogeneratedSchemaRefs(filteredAutogenlist);
+    await clearAutoGeneratedSchemaRefs(filteredAutoGenList);
 
     const schemaConfigs: SchemaConfiguration[] = [];
     const errors = [];
     const packages: Package[] = [];
-    for (const autogenlistConfig of filteredAutogenlist) {
+    for (const autoGenConfig of filteredAutoGenList) {
         let pkg = {
             path: ['schemas']
         } as Package;
         try {
-            const readme = await validateAndReturnReadmePath(localPath, autogenlistConfig.readmeFile || autogenlistConfig.basePath);
+            const readme = await validateAndReturnReadmePath(localPath, autoGenConfig.readmeFile || autoGenConfig.basePath);
             pkg.packageName = getPackageString(readme);
 
-            const newConfigs = await generateSchemas(readme, autogenlistConfig);
+            const newConfigs = await generateSchemas(readme, autoGenConfig);
             schemaConfigs.push(...newConfigs);
             pkg.result = 'succeeded';
         } catch(error) {
-            pkg.packageName = autogenlistConfig.basePath;
+            pkg.packageName = autoGenConfig.basePath;
             pkg.result = 'failed';
-            console.log(chalk.red(`Caught exception processing autogenlist entry ${autogenlistConfig.basePath}.`));
+            console.log(chalk.red(`Caught exception processing autogenlist entry ${autoGenConfig.basePath}.`));
             console.log(chalk.red(error));
 
             errors.push(error);
         }
         packages.push(pkg);
     }
 
-    await saveAutogeneratedSchemaRefs(flatten(schemaConfigs));
+    await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
 
     if (!!params.outputPath) {
         const outputPath = await resolveAbsolutePath(params.outputPath);

generator/cmd/generatesingle.ts

@@ -1,6 +1,6 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath, getPackageString } from '../specs';
-import { generateSchemas, saveAutogeneratedSchemaRefs } from '../generate';
+import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate';
 import process from 'process';
 import { findAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
@@ -24,20 +24,20 @@ executeSynchronous(async () => {
     }
 
     const schemaConfigs = [];
-    const autogenEntries = findAutogenEntries(basePath);
+    const autoGenEntries = findAutogenEntries(basePath);
 
-    if (autogenEntries.length === 0) {
+    if (autoGenEntries.length === 0) {
         const localSchemaConfigs = await generateSchemas(readme);
         schemaConfigs.push(...localSchemaConfigs);
     } else {
-        for (const autogenlistConfig of autogenEntries) {
+        for (const autoGenConfig of autoGenEntries) {
             console.log(`Using autogenlist config:`)
-            console.log(chalk.green(JSON.stringify(autogenlistConfig, null, 2)));
+            console.log(chalk.green(JSON.stringify(autoGenConfig, null, 2)));
 
-            const localSchemaConfigs = await generateSchemas(readme, autogenlistConfig);
+            const localSchemaConfigs = await generateSchemas(readme, autoGenConfig);
             schemaConfigs.push(...localSchemaConfigs);
         }
     }
 
-    await saveAutogeneratedSchemaRefs(schemaConfigs);
+    await saveAutoGeneratedSchemaRefs(schemaConfigs);
 });

generator/constants.ts

@@ -6,6 +6,7 @@ export const generatorRoot = path.resolve(__dirname, '../');
 export const specsRepoPath = path.join(os.tmpdir(), 'schm_azspc');
 export const specsRepoUri = 'https://github.com/azure/azure-rest-api-specs';
 export const specsRepoCommitHash = 'origin/master';
+export const pathRegex = /(microsoft\.\w+)[\\\/]\S*[\\\/](\d{4}-\d{2}-\d{2}(|-preview))[\\\/]/i;
 
 export const autoRestVerboseOutput = false;
 

generator/generate.ts

@@ -2,8 +2,9 @@ import path from 'path';
 import os from 'os';
 import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals, lowerCaseContains } from './utils';
 import * as constants from './constants';
+import { prepareReadme } from './specs';
 import chalk from 'chalk';
-import { ScopeType, AutogenlistConfig } from './models';
+import { ScopeType, AutoGenConfig } from './models';
 import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy } from 'lodash';
 
 const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
@@ -44,10 +45,12 @@ export async function getApiVersionsByNamespace(readme: string): Promise<Diction
     return output;
 }
 
-export async function generateSchemas(readme: string, autogenlistConfig?: AutogenlistConfig): Promise<SchemaConfiguration[]> {
+export async function generateSchemas(readme: string, autoGenConfig?: AutoGenConfig): Promise<SchemaConfiguration[]> {
+    await prepareReadme(readme, autoGenConfig);
+
     const apiVersionsByNamespace = pickBy(
         await getApiVersionsByNamespace(readme),
-        (_, key) => !autogenlistConfig || lowerCaseEquals(key, autogenlistConfig.namespace));
+        (_, key) => !autoGenConfig || lowerCaseEquals(key, autoGenConfig.namespace));
 
     const namespaces = keys(apiVersionsByNamespace);
 
@@ -63,7 +66,7 @@ export async function generateSchemas(readme: string, autogenlistConfig?: Autoge
                 continue;
             }
 
-            const generatedSchemaConfig = await handleGeneratedSchema(readme, schemaPath, autogenlistConfig);
+            const generatedSchemaConfig = await handleGeneratedSchema(readme, schemaPath, autoGenConfig);
 
             schemaConfigs.push(generatedSchemaConfig);
         }
@@ -75,19 +78,19 @@ export async function generateSchemas(readme: string, autogenlistConfig?: Autoge
     return schemaConfigs;
 }
 
-async function handleGeneratedSchema(readme: string, schemaPath: string, autogenlistConfig?: AutogenlistConfig) {
+async function handleGeneratedSchema(readme: string, schemaPath: string, autoGenConfig?: AutoGenConfig) {
     const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath))));
 
-    if (autogenlistConfig && autogenlistConfig.namespace.toLowerCase() !== namespace.toLowerCase()) {
+    if (autoGenConfig && autoGenConfig.namespace.toLowerCase() !== namespace.toLowerCase()) {
         throw new Error(`Encountered unexpected namespace ${namespace} in readme ${readme}`);
     }
 
     const apiVersion = path.basename(path.resolve(`${schemaPath}/..`));
 
-    const schemaConfig = await generateSchemaConfig(schemaPath, namespace, apiVersion, autogenlistConfig);
+    const schemaConfig = await generateSchemaConfig(schemaPath, namespace, apiVersion, autoGenConfig);
 
     const unknownScopeResources = schemaConfig.references.filter(x => x.scope & ScopeType.Unknown);
-    if (autogenlistConfig && unknownScopeResources.length > 0) {
+    if (autoGenConfig && unknownScopeResources.length > 0) {
         throw new Error(`Unable to determine scope for resource types ${unknownScopeResources.map(x => x.type).join(', ')} in readme ${readme}`);
     }
 
@@ -111,8 +114,7 @@ async function generateSchema(readme: string, tmpFolder: string) {
         `--use=@autorest/azureresourceschema@${constants.azureresourceschemaVersion}`,
         '--azureresourceschema',
         `--output-folder=${tmpFolder}`,
-        `--multiapi`,
-        '--title=none',
+        '--multiapi',
         '--pass-thru:subset-reducer',
         readme,
     ];
@@ -144,8 +146,8 @@ function getFilePathFromRef(schemaRef: string) {
     return path.resolve(path.join(constants.schemasBasePath, schemaUri.substring(constants.schemasBaseUri.length + 1)));
 }
 
-function assignScopesToUnknownReferences(knownReferences: SchemaReference[], unknownReferences: SchemaReference[], autogenlistConfig?: AutogenlistConfig) {
-    const resourceConfig = (autogenlistConfig || {}).resourceConfig || [];
+function assignScopesToUnknownReferences(knownReferences: SchemaReference[], unknownReferences: SchemaReference[], autoGenConfig?: AutoGenConfig) {
+    const resourceConfig = (autoGenConfig || {}).resourceConfig || [];
 
     for (const schemaRef of unknownReferences) {
         const config = resourceConfig.find(c => lowerCaseCompare(c.type, schemaRef.type) === 0);
@@ -169,14 +171,14 @@ function getSchemaFileName(namespace: string, suffix: string | undefined) {
     return `${namespace}.${suffix}.json`;
 }
 
-async function generateSchemaConfig(outputFile: string, namespace: string, apiVersion: string, autogenlistConfig?: AutogenlistConfig): Promise<SchemaConfiguration> {
-    namespace = autogenlistConfig?.namespace ?? namespace;
-    const suffix = autogenlistConfig?.suffix;
+async function generateSchemaConfig(outputFile: string, namespace: string, apiVersion: string, autoGenConfig?: AutoGenConfig): Promise<SchemaConfiguration> {
+    namespace = autoGenConfig?.namespace ?? namespace;
+    const suffix = autoGenConfig?.suffix;
     const relativePath = `${apiVersion}/${getSchemaFileName(namespace, suffix)}`;
 
     let output = await readJsonFile(outputFile);
-    if (autogenlistConfig?.postProcessor) {
-        autogenlistConfig?.postProcessor(namespace, apiVersion, output);
+    if (autoGenConfig?.postProcessor) {
+        autoGenConfig?.postProcessor(namespace, apiVersion, output);
 
         await writeJsonFile(outputFile, output);
     }
@@ -190,7 +192,7 @@ async function generateSchemaConfig(outputFile: string, namespace: string, apiVe
     ];
 
     const unknownReferences = getSchemaRefs(output, ScopeType.Unknown, 'unknown_resourceDefinitions');
-    assignScopesToUnknownReferences(knownReferences, unknownReferences, autogenlistConfig);
+    assignScopesToUnknownReferences(knownReferences, unknownReferences, autoGenConfig);
 
     const references = [
         ...knownReferences,
@@ -290,10 +292,10 @@ async function getCurrentTemplateRefs(scopeType: ScopeType, rootSchemaConfig: Ro
     return currentRefsOneOf.map(v => v['$ref']);
 }
 
-export async function clearAutogeneratedSchemaRefs(autogenlist: AutogenlistConfig[]) {
+export async function clearAutoGeneratedSchemaRefs(autoGenList: AutoGenConfig[]) {
     RootSchemaConfigs.forEach(async (rootSchemaConfig, scopeType) => {
         const currentRefs = await getCurrentTemplateRefs(scopeType, rootSchemaConfig);
-        const autogenlistedFiles = new Set(autogenlist.map(x => getSchemaFileName(x.namespace, x.suffix).toLowerCase()));
+        const autogenlistedFiles = new Set(autoGenList.map(x => getSchemaFileName(x.namespace, x.suffix).toLowerCase()));
         const schemasToRemove = [];
         const schemasByFilePath = groupBy(currentRefs, getFilePathFromRef);
         // clean up existing schemas to detect deletions
@@ -311,7 +313,7 @@ export async function clearAutogeneratedSchemaRefs(autogenlist: AutogenlistConfi
     });
 }
 
-export async function saveAutogeneratedSchemaRefs(schemaConfigs: SchemaConfiguration[]) {
+export async function saveAutoGeneratedSchemaRefs(schemaConfigs: SchemaConfiguration[]) {
     RootSchemaConfigs.forEach(async (rootSchemaConfig, scopeType) => {
         const refs = flatten(schemaConfigs
             .map(c => c.references

generator/models.ts

@@ -8,16 +8,25 @@ export enum ScopeType {
   Extension = 1 << 5,
 }
 
-export interface AutogenlistConfig {
+export interface AutoGenConfig {
   basePath: string,
   namespace: string,
   readmeFile?: string,
+  readmeTag?: ReadmeTag,
   suffix?: string,
-  resourceConfig?: AutogenlistResourceConfig[],
+  resourceConfig?: AutoGenResourceConfig[],
   postProcessor?: SchemaPostProcessor,
 }
 
-export interface AutogenlistResourceConfig {
+export interface ReadmeTag {
+  [apiVersion: string]: string[]
+}
+
+export interface CodeBlock {
+  readonly "input-file"?: ReadonlyArray<string>|string
+}
+
+export interface AutoGenResourceConfig {
   type: string,
   scopes?: ScopeType,
 }