Skip to content

Commit 4ee0f06

Browse files
webbertakkenwebbertakken
authored
Implement the queue (#21)
* Update the dispatch contract * Pseudo code the queue logic * Create jobs when unity versions get ingested * Create (many) jobs when repo versions get ingested * Cleanup * Unambiguous collection constants * Fix typo * Use strings for statuses... ...as they're stored in the database. * register repoVersionTrigger * Remove async from simple construct method * Yea, but put it in the right table, lol * Mark older jobs as superseded * Remove strange ids signature * Only mark any as superseded if list is not empty * Fix typo * Don't mark in-progress and comleted as superseded * Or rather, only leave completed in tact * Fix typo * Warn for skipped version instead of error * Simplify superseding * Fix off by base+hub * Improve messages * Improve messages further * only superseed "created" jobs * Fix typo * Break messages at spaces * First useful message for news channel * Better publication name * Add queueing logic * Fix typo * Match any 2xx statuses * Also reschedule base/hub images that failed to build * Better information * Just retry failures, np * Fix typo * More user-friendly copy * Add extra debugging * Fix typo * Fix typo * Improve logs * Implement ingeminator and incremental backoff * Implement maxRetries * Document subtleties * Fix overwrite bug * Clearer messages * Fix bug in haveAllBuildsForJobBeenPublished * Better copy * Update indexes * DangerouslyUpdateForProductionRelease
1 parent 070e8dd commit 4ee0f06

25 files changed

+918
-121
lines changed

‎firestore.indexes.json‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"queryScope": "COLLECTION",
66
"fields": [
77
{
8-
"fieldPath": "jobId",
8+
"fieldPath": "relatedJobId",
99
"order": "ASCENDING"
1010
},
1111
{
@@ -14,6 +14,42 @@
1414
}
1515
]
1616
},
17+
{
18+
"collectionGroup": "ciJobs",
19+
"queryScope": "COLLECTION",
20+
"fields": [
21+
{
22+
"fieldPath": "status",
23+
"order": "ASCENDING"
24+
},
25+
{
26+
"fieldPath": "editorVersionInfo.major",
27+
"order": "DESCENDING"
28+
},
29+
{
30+
"fieldPath": "editorVersionInfo.minor",
31+
"order": "DESCENDING"
32+
},
33+
{
34+
"fieldPath": "editorVersionInfo.patch",
35+
"order": "DESCENDING"
36+
}
37+
]
38+
},
39+
{
40+
"collectionGroup": "ciJobs",
41+
"queryScope": "COLLECTION",
42+
"fields": [
43+
{
44+
"fieldPath": "status",
45+
"order": "ASCENDING"
46+
},
47+
{
48+
"fieldPath": "repoVersionInfo.version",
49+
"order": "ASCENDING"
50+
}
51+
]
52+
},
1753
{
1854
"collectionGroup": "editorVersions",
1955
"queryScope": "COLLECTION",

‎functions/package-lock.json‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎functions/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
},
3030
"devDependencies": {
3131
"@types/jsdom": "^16.2.4",
32+
"@types/lodash": "^4.14.162",
3233
"@types/node": "^14.11.10",
3334
"@types/node-fetch": "^2.5.7",
3435
"@types/semver": "^7.3.4",

‎functions/src/api/manualWorkflowTrigger.ts‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@ export const manualWorkflowTrigger = functions.https.onRequest(
1818
// event_type: 'new_base_image_requested',
1919
// client_payload: {
2020
// jobId: 'someJobId',
21-
// buildId: 'someBuildId',
22-
// patchTag: '0.2.0',
23-
// minorTag: '0.2',
24-
// majorTag: '0',
21+
// repoVersionFull: '0.2.0',
22+
// repoVersionMinor: '0.2',
23+
// repoVersionMajor: '0',
2524
// },
2625
// });
2726

@@ -32,10 +31,9 @@ export const manualWorkflowTrigger = functions.https.onRequest(
3231
// event_type: 'new_hub_image_requested',
3332
// client_payload: {
3433
// jobId: 'someJobId',
35-
// buildId: 'someBuildId',
36-
// patchTag: '0.2.0',
37-
// minorTag: '0.2',
38-
// majorTag: '0',
34+
// repoVersionFull: '0.2.0',
35+
// repoVersionMinor: '0.2',
36+
// repoVersionMajor: '0',
3937
// },
4038
// });
4139

@@ -46,12 +44,11 @@ export const manualWorkflowTrigger = functions.https.onRequest(
4644
// event_type: 'new_editor_image_requested',
4745
// client_payload: {
4846
// jobId: 'someJobId',
49-
// buildId: 'someBuildId',
5047
// unityVersion: '2019.3.11f1',
5148
// changeSet: 'ceef2d848e70',
52-
// patchTag: '0.2.0',
53-
// minorTag: '0.2',
54-
// majorTag: '0',
49+
// repoVersionFull: '0.2.0',
50+
// repoVersionMinor: '0.2',
51+
// repoVersionMajor: '0',
5552
// },
5653
// });
5754

‎functions/src/api/reportPublication.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,25 @@ export const reportPublication = functions.https.onRequest(async (req: Request,
2424

2525
if (jobHasCompleted) {
2626
await CiJobs.markJobAsCompleted(jobId);
27-
const message = `New images published for ${jobId}.`;
27+
// Strip repo version from publication name
28+
let publicationName = jobId;
29+
const lastDashPos = publicationName.lastIndexOf('-');
30+
if (lastDashPos >= 1) {
31+
publicationName = publicationName.substr(0, lastDashPos);
32+
}
33+
34+
// Report new publications as news
35+
let message = '';
36+
if (dockerInfo.imageName === 'editor') {
37+
// i.e. [editor-2020.1.6f1]
38+
message = `Published ${publicationName} images.`;
39+
} else {
40+
// i.e. [hub] or [base]
41+
message = `New ${publicationName}-image published.`;
42+
}
2843
firebase.logger.info(message);
2944
if (!isDryRun) {
30-
await Discord.sendMessageToMaintainers(message);
45+
await Discord.sendNews(message);
3146
}
3247
}
3348

‎functions/src/config/discord.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,15 @@ export class Discord {
113113
messageSize -= 3;
114114
}
115115

116-
messages[i] = `${prefix}${message.substr(pointer, messageSize)}${suffix}`;
116+
// Break at spaces
117+
let maxMessage = message.substr(pointer, messageSize);
118+
const lastSpacePos = maxMessage.lastIndexOf(' ');
119+
if (lastSpacePos >= maxMessageSize - 250) {
120+
maxMessage = maxMessage.substr(pointer, lastSpacePos);
121+
messageSize = lastSpacePos;
122+
}
123+
124+
messages[i] = `${prefix}${maxMessage}${suffix}`;
117125
pointer += messageSize;
118126
}
119127

‎functions/src/config/github.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ const { 'private-key': privateKey, 'client-secret': clientSecret } = firebase.co
77

88
export class GitHub {
99
// https://octokit.github.io/rest.js/v18
10-
static async init() {
10+
static async init(): Promise<Octokit> {
1111
const appOctokit = new Octokit({
1212
authStrategy: createAppAuth,
1313
auth: {

‎functions/src/config/settings.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export const settings = {
2-
maxConcurrentBuilds: 1,
2+
maxConcurrentJobs: 5,
3+
maxFailuresPerBuild: 5,
34
discord: {
45
channels: {
56
news: '731947345478156391',

‎functions/src/cron/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { firebase, functions } from '../config/firebase';
33
import { Discord } from '../config/discord';
44
import { ingestUnityVersions } from '../logic/ingestUnityVersions';
55
import { ingestRepoVersions } from '../logic/ingestRepoVersions';
6+
import { scheduleBuildsFromTheQueue } from '../logic/buildQueue';
67

78
/**
89
* CPU-time for pubSub is not part of the free quota, so we'll keep it light weight.
@@ -28,4 +29,5 @@ export const trigger = functions
2829
const routineTasks = async () => {
2930
await ingestRepoVersions();
3031
await ingestUnityVersions();
32+
await scheduleBuildsFromTheQueue();
3133
};

‎functions/src/logic/buildQueue/generateBuildQueueFromNewVersionInfo.ts‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

0 commit comments

Comments
 (0)