Merge pull request #3016 from Azure/ant/fix_aks

Fix AKS autogeneration

Author: anthony-c-martin

.github/workflows/generate-single.yml

@@ -8,7 +8,7 @@ on:
         required: true
         default: 'main'
       single_path:
-        description: 'The path to generate types for (e.g. "compute", or "keyvault").'
+        description: 'The path to generate types for (e.g. "compute/resource-manager", or "keyvault/resource-manager").'
         required: true
 
 jobs:
@@ -40,7 +40,7 @@ jobs:
         run: |
           npm run generate-single -- \
             --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \
-            --base-path '${{ github.event.inputs.single_path }}/resource-manager'
+            --base-path '${{ github.event.inputs.single_path }}'
         working-directory: generator
 
       - name: Create Pull Request

.vscode/launch.json

@@ -26,7 +26,7 @@
       "args": [
         "${workspaceFolder}/generator/cmd/generatesingle.ts",
         "--base-path",
-        "${input:resourceProvider}/resource-manager"
+        "${input:resourceProvider}"
       ]
     },
     {
@@ -53,10 +53,10 @@
   ],
   "inputs": [
     {
-      "id": "resourceProvider",
+      "id": "basePath",
       "type": "promptString",
-      "default": "alertsmanagement",
-      "description": "The ResourceProvider name, e.g., compute, network"
+      "default": "alertsmanagement/resource-manager",
+      "description": "The path to generate types for (e.g. \"compute/resource-manager\", or \"keyvault/resource-manager\")."
     }
   ]
 }

generator/autogenlist.ts

@@ -17,6 +17,7 @@ import { postProcessor as azureStackHciPostProcessor } from './processors/Micros
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
 import { postProcessor as serviceFabricPostProcessor } from './processors/Microsoft.ServiceFabric';
 import { lowerCaseEquals } from './utils';
+import { detectProviderNamespaces } from './generate';
 
 // New providers are onboarded by default. The providers listed here are the only ones **not** onboarded.
 const disabledProviders: AutoGenConfig[] = [
@@ -110,10 +111,16 @@ const disabledProviders: AutoGenConfig[] = [
         disabledForAutogen: true
     },
     {
-        // Disabled temporally due to unsupported directory structure
-        basePath: 'containerservice/resource-manager',
+        basePath: 'containerservice/resource-manager/Microsoft.ContainerService/aks',
         namespace: 'Microsoft.ContainerService',
-        disabledForAutogen: true,
+        useNamespaceFromConfig: true,
+        suffix: 'Aks'
+    },
+    {
+        basePath: 'containerservice/resource-manager/Microsoft.ContainerService/fleet',
+        namespace: 'Microsoft.ContainerService',
+        useNamespaceFromConfig: true,
+        suffix: 'Fleet'
     },
 ];
 
@@ -1160,10 +1167,16 @@ export function findAutogenEntries(basePath: string): AutoGenConfig[] {
     return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath));
 }
 
-export function findOrGenerateAutogenEntries(basePath: string, namespaces: string[]): AutoGenConfig[] {
-    const entries = findAutogenEntries(basePath).filter(e => namespaces.some(ns => lowerCaseEquals(e.namespace, ns)));
+export async function findOrGenerateAutogenEntries(basePath: string, readme: string): Promise<AutoGenConfig[]> {
+    let entries = findAutogenEntries(basePath);
+    if (entries.some(e => e.useNamespaceFromConfig)) {
+        return entries;
+    }
+
+    const detectedNamespaces = await detectProviderNamespaces(readme);
+    entries = entries.filter(e => detectedNamespaces.some(ns => lowerCaseEquals(e.namespace, ns)));
 
-    for (const namespace of namespaces) {
+    for (const namespace of detectedNamespaces) {
         if (!entries.some(e => lowerCaseEquals(e.namespace, namespace))) {
             // Generate configuration for any RPs not explicitly declared in the autogen list
             entries.push({

generator/cmd/generateall.ts

@@ -2,10 +2,10 @@
 // Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
-import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
+import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs } from '../generate';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import colors from 'colors';
-import { flatten, keys } from 'lodash';
+import { flatten } from 'lodash';
 import { executeSynchronous, chunker, writeJsonFile } from '../utils';
 import { Package } from '../models';
 import yargs from 'yargs';
@@ -72,8 +72,7 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         try {
             const readme = validateAndReturnReadmePath(localPath, basePath);
-            const namespaces = keys(await getApiVersionsByNamespace(readme));
-            let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces)
+            let filteredAutoGenList = (await findOrGenerateAutogenEntries(basePath, readme))
                 .filter(x => x.disabledForAutogen !== true);
 
             if (args['readme-files']) {

generator/cmd/generateonboardedreport.ts

@@ -4,8 +4,7 @@ import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous, writeJsonFile, safeMkdir } from '../utils';
-import { getApiVersionsByNamespace } from '../generate';
-import { keys, partition } from 'lodash';
+import { partition } from 'lodash';
 import path from 'path';
 
 executeSynchronous(async () => {
@@ -15,8 +14,7 @@ executeSynchronous(async () => {
 
     for (const basePath of basePaths) {
         const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath);
-        const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+        const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
         const [unautogened, autogened] = partition(
             autogenlistEntries,

generator/cmd/generatesingle.ts

@@ -2,10 +2,9 @@
 // Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
-import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
+import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import colors from 'colors';
-import { keys } from 'lodash';
 import { executeSynchronous } from '../utils';
 import yargs from 'yargs';
 
@@ -34,8 +33,7 @@ executeSynchronous(async () => {
     }
 
     const schemaConfigs = [];
-    const namespaces = keys(await getApiVersionsByNamespace(readme));
-    const autoGenEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+    const autoGenEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
     for (const autoGenConfig of autoGenEntries) {
         if (autoGenConfig.disabledForAutogen === true) {

generator/cmd/listbasepaths.ts

@@ -5,16 +5,14 @@ import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs
 import colors from 'colors';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous } from '../utils';
-import { getApiVersionsByNamespace } from '../generate';
-import { keys, partition } from 'lodash';
+import { partition } from 'lodash';
 
 executeSynchronous(async () => {
     const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
 
     for (const basePath of basePaths) {
         const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath);
-        const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+        const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
         const [unautogened, autogened] = partition(
             autogenlistEntries,

generator/constants.ts

@@ -8,8 +8,6 @@ export const generatorRoot = path.resolve(__dirname, '../');
 export const specsRepoPath = path.join(os.tmpdir(), 'schm_azspc');
 export const specsRepoUri = 'https://github.com/azure/azure-rest-api-specs';
 export const specsRepoCommitHash = 'origin/main';
-// eslint-disable-next-line no-useless-escape
-export const pathRegex = /(microsoft\.\w+|NGINX.NGINXPLUS|DYNATRACE.OBSERVABILITY)[\\\/]\S*[\\\/](\d{4}-\d{2}-\d{2}(|-preview))[\\\/]/i;
 
 export const autoRestVerboseOutput = false;
 

generator/generate.ts

@@ -2,15 +2,15 @@
 // Licensed under the MIT License.
 import path from 'path';
 import os from 'os';
-import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals, lowerCaseContains } from './utils';
+import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals } from './utils';
 import * as constants from './constants';
 import { prepareReadme } from './specs';
 import colors from 'colors';
 import { ScopeType, AutoGenConfig } from './models';
-import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy } from 'lodash';
+import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference } from 'lodash';
 
 const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
-const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
+export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
 
 export interface SchemaConfiguration {
     references: SchemaReference[];
@@ -35,27 +35,17 @@ const RootSchemaConfigs: Map<ScopeType, RootSchemaConfiguration> = new Map([
     [ScopeType.ManagementGroup, constants.managementGroupRootSchema]
 ]);
 
-export async function getApiVersionsByNamespace(readme: string): Promise<Dictionary<string[]>> {
+export async function detectProviderNamespaces(readme: string) {
     const searchPath = path.resolve(`${readme}/..`);
-    const apiVersionPaths = await findDirRecursive(searchPath, p => path.basename(p).match(apiVersionRegex) !== null);
-
-    const output: Dictionary<string[]> = {};
-    for (const [namespace, , apiVersion] of apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep))) {
-        output[namespace] = [...(output[namespace] ?? []), apiVersion];
-    }
 
-    return output;
+    // To try and detect possible provider namespaces, assume a folder structure of <provider>/preview|stable/<api-version>/..., based on convention
+    const apiVersionPaths = await findDirRecursive(searchPath, p => path.basename(p).match(apiVersionRegex) !== null);
+    return uniq(apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep)[0]));
 }
 
-export async function generateSchemas(readme: string, autoGenConfig?: AutoGenConfig): Promise<SchemaConfiguration[]> {
+export async function generateSchemas(readme: string, autoGenConfig: AutoGenConfig): Promise<SchemaConfiguration[]> {
     await prepareReadme(readme, autoGenConfig);
 
-    const apiVersionsByNamespace = pickBy(
-        await getApiVersionsByNamespace(readme),
-        (_, key) => !autoGenConfig || lowerCaseEquals(key, autoGenConfig.namespace));
-
-    const namespaces = keys(apiVersionsByNamespace);
-
     const schemaConfigs: SchemaConfiguration[] = [];
     const tmpFolder = path.join(os.tmpdir(), Math.random().toString(36).substr(2));
 
@@ -64,7 +54,7 @@ export async function generateSchemas(readme: string, autoGenConfig?: AutoGenCon
 
         for (const schemaPath of generatedSchemas) {
             const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath))));
-            if (!lowerCaseContains(namespaces, namespace)) {
+            if (!lowerCaseEquals(autoGenConfig.namespace, namespace)) {
                 continue;
             }
 

generator/models.ts

@@ -14,6 +14,7 @@ export interface AutoGenConfig {
   disabledForAutogen?: true,
   basePath: string,
   namespace: string,
+  useNamespaceFromConfig?: boolean,
   readmeFile?: string,
   readmeTag?: ReadmeTag,
   suffix?: string,