Ingest repo versions (#17)

webbertakken · web-flow · commit a67abaeb3974 · 2020-10-17T16:49:42.000+02:00
* Alert that no versions have been ingested yet.

* Ingest repository versions

* Move types to development deps

* Fix typo

* Update database indexes

* Let us know which ones
diff --git a/firestore.indexes.json b/firestore.indexes.json
@@ -1,7 +1,7 @@
 {
   "indexes": [
     {
-      "collectionGroup": "repoVersions",
+      "collectionGroup": "editorVersions",
       "queryScope": "COLLECTION",
       "fields": [
         {
@@ -19,7 +19,7 @@
       ]
     },
     {
-      "collectionGroup": "unityVersions",
+      "collectionGroup": "repoVersions",
       "queryScope": "COLLECTION",
       "fields": [
         {
diff --git a/functions/package-lock.json b/functions/package-lock.json
diff --git a/functions/package.json b/functions/package.json
@@ -24,12 +24,14 @@
     "httpie": "^1.1.2",
     "jsdom": "^16.4.0",
     "lodash": "^4.17.20",
-    "node-fetch": "^2.6.1"
+    "node-fetch": "^2.6.1",
+    "semver": "^7.3.2"
   },
   "devDependencies": {
     "@types/jsdom": "^16.2.4",
     "@types/node": "^14.11.5",
     "@types/node-fetch": "^2.5.7",
+    "@types/semver": "^7.3.4",
     "@typescript-eslint/eslint-plugin": "^3.9.1",
     "@typescript-eslint/parser": "^3.8.0",
     "eslint": "^7.6.0",
diff --git a/functions/src/api/repoVerions.ts b/functions/src/api/repoVerions.ts
@@ -1,7 +1,7 @@
 import { Request } from 'firebase-functions/lib/providers/https';
 import { Response } from 'express-serve-static-core';
 import { firebase, functions } from '../config/firebase';
-import { RepoVersionInfo } from '../model/repoVersions';
+import { RepoVersionInfo } from '../model/repoVersionInfo';
 
 export const repoVersions = functions.https.onRequest(
   async (request: Request, response: Response) => {
diff --git a/functions/src/api/reportNewBuild.ts b/functions/src/api/reportNewBuild.ts
@@ -6,7 +6,7 @@ import { BuildInfo, CiBuilds, ImageType } from '../model/ciBuilds';
 import { CiJobs } from '../model/ciJobs';
 import { Discord } from '../config/discord';
 import { EditorVersionInfo } from '../model/editorVersionInfo';
-import { RepoVersionInfo } from '../model/repoVersions';
+import { RepoVersionInfo } from '../model/repoVersionInfo';
 
 export const reportNewBuild = functions.https.onRequest(async (req: Request, res: Response) => {
   try {
diff --git a/functions/src/cron/index.ts b/functions/src/cron/index.ts
@@ -2,6 +2,7 @@ import { EventContext } from 'firebase-functions';
 import { firebase, functions } from '../config/firebase';
 import { Discord } from '../config/discord';
 import { ingestUnityVersions } from '../logic/ingestUnityVersions';
+import { ingestRepoVersions } from '../logic/ingestRepoVersions';
 
 /**
  * CPU-time for pubSub is not part of the free quota, so we'll keep it light weight.
@@ -24,5 +25,6 @@ export const trigger = functions.pubsub
   });
 
 const routineTasks = async () => {
+  await ingestRepoVersions();
   await ingestUnityVersions();
 };
diff --git a/functions/src/logic/ingestRepoVersions/index.ts b/functions/src/logic/ingestRepoVersions/index.ts
@@ -0,0 +1,21 @@
+import { firebase } from '../../config/firebase';
+import { Discord } from '../../config/discord';
+import { scrapeVersions } from './scrapeVersions';
+import { updateDatabase } from './updateDatabase';
+
+export const ingestRepoVersions = async () => {
+  try {
+    const scrapedInfoList = await scrapeVersions();
+    firebase.logger.info('Found versions', scrapedInfoList);
+
+    await updateDatabase(scrapedInfoList);
+  } catch (err) {
+    const message = `
+        Something went wrong while importing repository versions for unity-ci/docker:
+        ${err.message} (${err.status})\n${err.stackTrace}
+      `;
+
+    firebase.logger.error(message);
+    await Discord.sendAlert(message);
+  }
+};
diff --git a/functions/src/logic/ingestRepoVersions/scrapeVersions.ts b/functions/src/logic/ingestRepoVersions/scrapeVersions.ts
@@ -0,0 +1,47 @@
+import semver from 'semver';
+import { RepoVersionInfo } from '../../model/repoVersionInfo';
+import { GitHub } from '../../config/github';
+
+export const scrapeVersions = async (): Promise<RepoVersionInfo[]> => {
+  const gitHub = await GitHub.init();
+
+  const releases = await gitHub.repos.listReleases({
+    owner: 'unity-ci',
+    repo: 'docker',
+  });
+
+  const versions = releases.data.map((release) => {
+    const {
+      id,
+      url,
+      name,
+      body: description,
+      tag_name: tagName,
+      author: { login: author },
+      target_commitish: commitIsh,
+    } = release;
+
+    const version = semver.valid(semver.coerce(tagName));
+    if (!version) {
+      throw new Error("Assumed versions to always be parsable, but they're not.");
+    }
+    const major = semver.major(version);
+    const minor = semver.major(version);
+    const patch = semver.major(version);
+
+    return {
+      version,
+      major,
+      minor,
+      patch,
+      id,
+      name,
+      description,
+      author,
+      commitIsh,
+      url,
+    };
+  });
+
+  return versions;
+};
diff --git a/functions/src/logic/ingestRepoVersions/updateDatabase.ts b/functions/src/logic/ingestRepoVersions/updateDatabase.ts
@@ -0,0 +1,54 @@
+import { isMatch } from 'lodash';
+import { RepoVersionInfo } from '../../model/repoVersionInfo';
+import { firebase } from '../../config/firebase';
+import { Discord } from '../../config/discord';
+
+const plural = (amount: number) => {
+  return amount === 1 ? 'version' : 'versions';
+};
+
+export const updateDatabase = async (ingestedInfoList: RepoVersionInfo[]): Promise<void> => {
+  const existingInfoList = await RepoVersionInfo.getAll();
+
+  const newVersions: RepoVersionInfo[] = [];
+  const updatedVersions: RepoVersionInfo[] = [];
+
+  ingestedInfoList.forEach((ingestedInfo: RepoVersionInfo) => {
+    const { version } = ingestedInfo;
+    const existingVersion = existingInfoList.find((info) => info.version === version);
+
+    if (!existingVersion) {
+      newVersions.push(ingestedInfo);
+      return;
+    }
+
+    if (!isMatch(existingVersion, ingestedInfo)) {
+      updatedVersions.push(ingestedInfo);
+      return;
+    }
+  });
+
+  let message = '';
+
+  if (newVersions.length >= 1) {
+    await RepoVersionInfo.createMany(newVersions);
+    message += `
+      ${newVersions.length} new repository ${plural(newVersions.length)} detected.
+      (${newVersions.map((version) => version.version).join(', ')}`;
+  }
+
+  if (updatedVersions.length >= 1) {
+    await RepoVersionInfo.updateMany(updatedVersions);
+    message += `
+      ${updatedVersions.length} updated repository ${plural(updatedVersions.length)} detected.
+      (${updatedVersions.map((version) => version.version).join(', ')})`;
+  }
+
+  message = message.trimEnd();
+  if (message.length >= 1) {
+    firebase.logger.info(message);
+    await Discord.sendMessageToMaintainers(message);
+  } else {
+    firebase.logger.info('Database is up-to-date. (no updated repo versions found)');
+  }
+};
diff --git a/functions/src/logic/ingestUnityVersions/updateDatabase.ts b/functions/src/logic/ingestUnityVersions/updateDatabase.ts
@@ -1,7 +1,7 @@
-import { EditorVersionInfo } from '../../model/editorVersionInfo';
 import { isMatch } from 'lodash';
 import { firebase } from '../../config/firebase';
 import { Discord } from '../../config/discord';
+import { EditorVersionInfo } from '../../model/editorVersionInfo';
 
 const plural = (amount: number) => {
   return amount === 1 ? 'version' : 'versions';
@@ -32,12 +32,16 @@ export const updateDatabase = async (ingestedInfoList: EditorVersionInfo[]): Pro
 
   if (newVersions.length >= 1) {
     await EditorVersionInfo.createMany(newVersions);
-    message += `${newVersions.length} new ${plural(newVersions.length)} detected. `;
+    message += `
+      ${newVersions.length} new Unity editor ${plural(newVersions.length)} detected.
+      (${newVersions.map((version) => version.version).join(', ')}`;
   }
 
   if (updatedVersions.length >= 1) {
     await EditorVersionInfo.updateMany(updatedVersions);
-    message += `${updatedVersions.length} updated ${plural(updatedVersions.length)} detected. `;
+    message += `
+      ${updatedVersions.length} updated Unity editor ${plural(updatedVersions.length)} detected.
+      (${updatedVersions.map((version) => version.version).join(', ')})`;
   }
 
   message = message.trimEnd();
diff --git a/functions/src/model/ciJobs.ts b/functions/src/model/ciJobs.ts
@@ -2,7 +2,7 @@ import { db, admin } from '../config/firebase';
 import { EditorVersionInfo } from './editorVersionInfo';
 import FieldValue = admin.firestore.FieldValue;
 import Timestamp = admin.firestore.Timestamp;
-import { RepoVersionInfo } from './repoVersions';
+import { RepoVersionInfo } from './repoVersionInfo';
 import { ImageType } from './ciBuilds';
 
 const COLLECTION = 'ciJobs';
diff --git a/functions/src/model/ciVersionInfo.ts b/functions/src/model/ciVersionInfo.ts
@@ -1,7 +1,7 @@
 import { db, admin, firebase } from '../config/firebase';
 import Timestamp = admin.firestore.Timestamp;
 import { EditorVersionInfo } from './editorVersionInfo';
-import { RepoVersionInfo } from './repoVersions';
+import { RepoVersionInfo } from './repoVersionInfo';
 
 const COLLECTION = 'builtVersions';
 
diff --git a/functions/src/model/editorVersionInfo.ts b/functions/src/model/editorVersionInfo.ts
@@ -1,7 +1,7 @@
 import { db, admin, firebase } from '../config/firebase';
 import Timestamp = admin.firestore.Timestamp;
 
-const COLLECTION = 'unityVersions';
+const COLLECTION = 'editorVersions';
 
 export interface EditorVersionInfo {
   version: string;
@@ -42,11 +42,11 @@ export class EditorVersionInfo {
     return snapshot.docs.map((doc) => doc.data()) as EditorVersionInfo[];
   };
 
-  static createMany = async (versionInfoList: EditorVersionInfo[]) => {
+  static createMany = async (editorVersionList: EditorVersionInfo[]) => {
     try {
       const batch = db.batch();
 
-      versionInfoList.forEach((versionInfo) => {
+      editorVersionList.forEach((versionInfo) => {
         const { version } = versionInfo;
 
         const ref = db.collection(COLLECTION).doc(version);
@@ -56,7 +56,7 @@ export class EditorVersionInfo {
 
       await batch.commit();
     } catch (err) {
-      firebase.logger.error('Error occurred during batch commit of new version', err);
+      firebase.logger.error('Error occurred during batch commit of new editor versions', err);
     }
   };
 
@@ -74,7 +74,7 @@ export class EditorVersionInfo {
 
       await batch.commit();
     } catch (err) {
-      firebase.logger.error('Error occurred during batch commit of new version', err);
+      firebase.logger.error('Error occurred during batch commit of new editor versions', err);
     }
   };
 }
diff --git a/functions/src/model/repoVersionInfo.ts b/functions/src/model/repoVersionInfo.ts

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"indexes": [`
`3`	`3`	`{`
`4`		`- "collectionGroup": "repoVersions",`
	`4`	`+ "collectionGroup": "editorVersions",`
`5`	`5`	`"queryScope": "COLLECTION",`
`6`	`6`	`"fields": [`
`7`	`7`	`{`
`@@ -19,7 +19,7 @@`
`19`	`19`	`]`
`20`	`20`	`},`
`21`	`21`	`{`
`22`		`- "collectionGroup": "unityVersions",`
	`22`	`+ "collectionGroup": "repoVersions",`
`23`	`23`	`"queryScope": "COLLECTION",`
`24`	`24`	`"fields": [`
`25`	`25`	`{`