Skip to content

Commit faeadc8

Browse files
jorgecotillojorgecotillo
committed
initial update
1 parent d6a8a89 commit faeadc8

File tree

2 files changed

+244
-9
lines changed

2 files changed

+244
-9
lines changed

‎.github/workflows/generate-schemas-batch.yml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
name: Generate Types
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
api_specs_ref:
7+
description: 'Git ref or full SHA for https://github.com/Azure/azure-rest-api-specs.'
8+
required: true
9+
default: 'main'
10+
11+
env:
12+
SUMMARY_LOG_PATH: tmp/summary/summary.log
13+
AUTOGENERATE_BRANCH_NAME: autogenerate
14+
jobs:
15+
batch-0:
16+
name: Update Types
17+
runs-on: ubuntu-latest
18+
steps:
19+
- uses: actions/checkout@v2.3.5
20+
21+
- name: Clone azure-rest-api-specs
22+
uses: actions/checkout@v2.3.5
23+
with:
24+
repository: Azure/azure-rest-api-specs
25+
path: workflow-temp/azure-rest-api-specs
26+
ref: ${{ github.event.inputs.api_specs_ref }}
27+
28+
- name: Install generator npm packages
29+
run: npm ci
30+
working-directory: generator
31+
32+
- id: generate
33+
name: Run generator
34+
run: |
35+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 0
36+
working-directory: generator
37+
- name: Commit & push update
38+
run: |
39+
git add --all schemas
40+
git commit -m "Autogenerate schemas"
41+
git push origin $AUTOGENERATE_BRANCH_NAME
42+
- name: Upload summary log
43+
uses: actions/upload-artifact@v2
44+
with:
45+
name: summary-log
46+
path: ${{ env.SUMMARY_LOG_PATH }}
47+
batch-1:
48+
needs: batch-0
49+
name: Update Types
50+
runs-on: ubuntu-latest
51+
steps:
52+
- uses: actions/checkout@v2.3.5
53+
54+
- name: Clone azure-rest-api-specs
55+
uses: actions/checkout@v2.3.5
56+
with:
57+
repository: Azure/azure-rest-api-specs
58+
path: workflow-temp/azure-rest-api-specs
59+
ref: ${{ github.event.inputs.api_specs_ref }}
60+
61+
- name: Install generator npm packages
62+
run: npm ci
63+
working-directory: generator
64+
65+
- name: Download summary log artifact
66+
uses: actions/download-artifact@v2
67+
with:
68+
name: summary-log
69+
path: ${{ env.SUMMARY_LOG_PATH }}
70+
71+
- id: generate
72+
name: Run generator
73+
run: |
74+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 1
75+
working-directory: generator
76+
77+
- name: Commit & push update
78+
run: |
79+
git add --all schemas
80+
git commit -m "Autogenerate schemas"
81+
git push origin $AUTOGENERATE_BRANCH_NAME
82+
83+
- name: Upload summary log
84+
uses: actions/upload-artifact@v2
85+
with:
86+
name: summary-log
87+
path: ${{ env.SUMMARY_LOG_PATH }}
88+
batch-2:
89+
needs: batch-1
90+
name: Update Types
91+
runs-on: ubuntu-latest
92+
steps:
93+
- uses: actions/checkout@v2.3.5
94+
95+
- name: Clone azure-rest-api-specs
96+
uses: actions/checkout@v2.3.5
97+
with:
98+
repository: Azure/azure-rest-api-specs
99+
path: workflow-temp/azure-rest-api-specs
100+
ref: ${{ github.event.inputs.api_specs_ref }}
101+
102+
- name: Install generator npm packages
103+
run: npm ci
104+
working-directory: generator
105+
106+
- name: Download summary log artifact
107+
uses: actions/download-artifact@v2
108+
with:
109+
name: summary-log
110+
path: ${{ env.SUMMARY_LOG_PATH }}
111+
112+
- id: generate
113+
name: Run generator
114+
run: |
115+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 2
116+
working-directory: generator
117+
118+
- name: Commit & push update
119+
run: |
120+
git add --all schemas
121+
git commit -m "Autogenerate schemas"
122+
git push origin $AUTOGENERATE_BRANCH_NAME
123+
124+
- name: Upload summary log
125+
uses: actions/upload-artifact@v2
126+
with:
127+
name: summary-log
128+
path: ${{ env.SUMMARY_LOG_PATH }}
129+
batch-3:
130+
needs: batch-2
131+
name: Update Types
132+
runs-on: ubuntu-latest
133+
steps:
134+
- uses: actions/checkout@v2.3.5
135+
136+
- name: Clone azure-rest-api-specs
137+
uses: actions/checkout@v2.3.5
138+
with:
139+
repository: Azure/azure-rest-api-specs
140+
path: workflow-temp/azure-rest-api-specs
141+
ref: ${{ github.event.inputs.api_specs_ref }}
142+
143+
- name: Install generator npm packages
144+
run: npm ci
145+
working-directory: generator
146+
147+
- name: Download summary log artifact
148+
uses: actions/download-artifact@v2
149+
with:
150+
name: summary-log
151+
path: ${{ env.SUMMARY_LOG_PATH }}
152+
153+
- id: generate
154+
name: Run generator
155+
run: |
156+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 3
157+
158+
summary="$(<$SUMMARY_LOG_PATH)"
159+
summary="${summary//'%'/'%25'}"
160+
summary="${summary//$'\n'/'%0A'}"
161+
summary="${summary//$'\r'/'%0D'}"
162+
echo "::set-output name=summary::$summary"
163+
working-directory: generator
164+
165+
- id: get_swagger_gh_uri
166+
name: Get GitHub URI for azure-rest-api-specs
167+
run: |
168+
git_sha=`git rev-parse HEAD`
169+
echo "::set-output name=gh_uri::https://github.com/Azure/azure-rest-api-specs/tree/$git_sha"
170+
working-directory: workflow-temp/azure-rest-api-specs
171+
172+
- name: Create Pull Request
173+
uses: peter-evans/create-pull-request@v3.10.1
174+
with:
175+
committer: GitHub <noreply@github.com>
176+
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
177+
signoff: false
178+
branch: autogenerate
179+
branch-suffix: short-commit-hash
180+
delete-branch: true
181+
title: |
182+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
183+
body: |
184+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
185+
186+
Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
187+
188+
Summary
189+
${{ steps.generate.outputs.summary }}
190+
commit-message: |
191+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
192+
193+
Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
194+
labels: autogenerate
195+
draft: false

‎generator/cmd/generateall.ts‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,52 @@ import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolve
55
import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
66
import { findOrGenerateAutogenEntries } from '../autogenlist';
77
import chalk from 'chalk';
8-
import { flatten, keys } from 'lodash';
8+
import { flatten, keys, sum } from 'lodash';
99
import { executeSynchronous, chunker, writeJsonFile } from '../utils';
1010
import { Package } from '../models';
1111
import yargs from 'yargs';
12+
import path from 'path/posix';
13+
14+
import { createWriteStream, existsSync } from 'fs';
15+
import { readdir, stat, rmdir, mkdir, rm, writeFile, readFile, copyFile } from 'fs/promises';
16+
import stripAnsi from 'strip-ansi';
1217

1318
const argsConfig = yargs
1419
.strict()
1520
.option('batch-count', { type: 'number', desc: 'If running in batch mode, the total number of batch jobs running' })
1621
.option('batch-index', { type: 'number', desc: 'If running in batch mode, the index of this batch job' })
1722
.option('local-path', { type: 'string', desc: 'The local path to the azure-rest-api-specs repo' })
1823
.option('readme-files', { type: 'array', desc: 'The list of readme.md files to generate schemas for' })
19-
.option('output-path', { type: 'string', desc: 'The base path to save schema output' });
24+
.option('output-path', { type: 'string', desc: 'The base path to save schema output' })
25+
.option('summary-log-path', { type: 'string', desc: 'The path to store generation summary information. File will be saved in md format.' });
26+
27+
interface ILogger {
28+
out: (data: string) => void;
29+
}
2030

2131
executeSynchronous(async () => {
2232
const args = await argsConfig.parseAsync();
2333

2434
let basePaths;
2535
let localPath = args['local-path'];
36+
let summaryPath = args['summary-log-path'];
37+
2638
if (!localPath) {
27-
localPath = constants.specsRepoPath;
39+
localPath = 'C:\\tmp'; //constants.specsRepoPath;
2840
basePaths = await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
2941
} else {
3042
localPath = await resolveAbsolutePath(localPath);
3143
basePaths = await generateBasePaths(localPath);
3244
}
3345

46+
if (!summaryPath) {
47+
// generate default full path
48+
summaryPath = path.join(constants.specsRepoPath, 'summary.log');
49+
}
50+
51+
// resolve absolute path
52+
summaryPath = await resolveAbsolutePath(summaryPath);
53+
3454
if (args['batch-count'] !== undefined && args['batch-index'] !== undefined) {
3555
basePaths = chunker(basePaths, args['batch-count'])[args['batch-index']];
3656
}
@@ -39,6 +59,8 @@ executeSynchronous(async () => {
3959
const errors = [];
4060
const packages: Package[] = [];
4161

62+
const summaryLogger = await getLogger(summaryPath);
63+
4264
for (const basePath of basePaths) {
4365
const readme = await validateAndReturnReadmePath(localPath, basePath);
4466
const namespaces = keys(await getApiVersionsByNamespace(readme));
@@ -76,7 +98,15 @@ executeSynchronous(async () => {
7698
console.log(chalk.red(`Caught exception processing autogenlist entry ${autoGenConfig.basePath}.`));
7799
console.log(chalk.red(error));
78100

79-
errors.push(error);
101+
// Use markdown formatting as this summary will be included in the PR description
102+
logOut(summaryLogger,
103+
`<details>
104+
<summary>Failed to generate types for path '${basePath}'</summary>
105+
\`\`\`
106+
${error}
107+
\`\`\`
108+
</details>
109+
`);
80110
}
81111
packages.push(pkg);
82112
}
@@ -87,10 +117,20 @@ executeSynchronous(async () => {
87117
if (args['output-path']) {
88118
const outputPath = await resolveAbsolutePath(args['output-path']);
89119
await writeJsonFile(outputPath, { packages });
90-
} else {
91-
if (errors.length > 0) {
92-
throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
93-
}
94120
}
95-
96121
});
122+
123+
function logOut(logger: ILogger, line: string) {
124+
logger.out(`${line}\n`);
125+
}
126+
127+
async function getLogger(logFilePath: string): Promise<ILogger> {
128+
const logFileStream = createWriteStream(logFilePath, { flags: 'a' });
129+
130+
return {
131+
out: (data: string) => {
132+
process.stdout.write(data);
133+
logFileStream.write(stripAnsi(data));
134+
}
135+
};
136+
}

0 commit comments

Comments
 (0)