Fix Unity Scrape Method (#51)

* Improved deployment tests. More github auth fixes

* More function deployment testing

* Fix formatting

* Force curl to print error info

* Remove stray return

* Test new scrape method. Add scrape test to testFunction

* Only include f releases. Print all unity versions on response for test function

* Fix index

* Remove duplicate versions

* Match original order

* Filter out Unity 5

.github/workflows/main.yml

@@ -58,8 +58,13 @@ jobs:
           args: deploy --only functions:testFunction
         env:
           GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'
+
+      - name: Call Test Function
+        run: curl -f -s -S -X POST https://testfunction-wbe4ukn6tq-ey.a.run.app
+
       - name: Cleanup Firebase Test
         uses: w9jds/[email protected]
+        if: always()
         with:
           args: functions:delete testFunction --force
         env:
@@ -86,6 +91,7 @@ jobs:
           GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}'
       - name: Cleanup Firebase Test
         uses: w9jds/[email protected]
+        if: always()
         with:
           args: functions:delete testFunction --force
         env:

functions/src/api/testFunction.ts

@@ -1,6 +1,10 @@
 import { onRequest, Request } from 'firebase-functions/v2/https';
 import { Response } from 'express-serve-static-core';
 import { defineSecret } from 'firebase-functions/params';
+import { scrapeVersions } from '../logic/ingestRepoVersions/scrapeVersions';
+import { scrapeVersions as scrapeUnityVersions } from '../logic/ingestUnityVersions/scrapeVersions';
+
+import { Discord } from '../service/discord';
 
 const discordToken = defineSecret('DISCORD_TOKEN');
 const githubPrivateKeyConfigSecret = defineSecret('GITHUB_PRIVATE_KEY');
@@ -9,14 +13,49 @@ const internalToken = defineSecret('INTERNAL_TOKEN');
 
 export const testFunction = onRequest(
   {
-    // Passing secrets so that test deployments verify that the secrets are correctly set.
+    // Passing all secrets so that test deployments verify that the secrets are correctly set.
     secrets: [
       discordToken,
       githubPrivateKeyConfigSecret,
       githubClientSecretConfigSecret,
       internalToken,
     ],
   },
-  // eslint-disable-next-line @typescript-eslint/no-empty-function
-  async (request: Request, response: Response) => {},
+  async (request: Request, response: Response) => {
+    // Run all non-sensitive functions to verify that the deployment is working.
+    const discordClient = new Discord();
+    let info = 'Ok';
+    let code = 200;
+
+    try {
+      await discordClient.init(discordToken.value());
+
+      const versions = await scrapeVersions(
+        githubPrivateKeyConfigSecret.value(),
+        githubClientSecretConfigSecret.value(),
+      );
+
+      if (versions.length === 0) {
+        throw new Error('No versions were found.');
+      }
+
+      const unityVersions = await scrapeUnityVersions();
+      if (unityVersions.length === 0) {
+        throw new Error('No Unity versions were found.');
+      }
+
+      info = `Found ${versions.length} repo versions and ${
+        unityVersions.length
+      } Unity versions. Unity Versions: \n${unityVersions
+        .map((unity) => `${unity.version}:${unity.changeSet}`)
+        .join('\n')}`;
+    } catch (error: any) {
+      info = error.message;
+      code = 500;
+    } finally {
+      await discordClient.disconnect();
+    }
+
+    response.status(code).send(info);
+  },
 );

functions/src/config/settings.ts

@@ -21,6 +21,7 @@ export const settings = {
   github: {
     auth: {
       appId: 84327,
+      installationId: 12321333,
       clientId: 'Iv1.fa93dce6a47c9357',
     },
   },

functions/src/logic/ingestUnityVersions/scrapeVersions.ts

@@ -2,29 +2,45 @@ import { getDocumentFromUrl } from '../utils/get-document-from-url';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
 
 const UNITY_ARCHIVE_URL = 'https://unity.com/releases/editor/archive';
+const unity_version_regex = /unityhub:\/\/(\d+)\.(\d+)\.(\d+[a-zA-Z]\d+)\/(\w+)/g;
 
-/**
- * Based on https://github.com/BLaZeKiLL/unity-scraper
- */
 export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
   const document = await getDocumentFromUrl(UNITY_ARCHIVE_URL);
 
-  const links = Array.from(document.querySelectorAll('.release-links div:first-child a[href]'));
-  const hrefs = links.map((a) => a.getAttribute('href')) as string[];
+  const scripts = document.querySelectorAll('script');
 
-  const versionInfoList = hrefs.map((href) => {
-    const info = href.replace('unityhub://', '');
-    const [version, changeSet] = info.split('/');
-    const [major, minor, patch] = version.split('.');
+  for (const script of scripts) {
+    if (script.textContent) {
+      const matches = [...script.textContent.matchAll(unity_version_regex)];
+      if (matches.length > 0) {
+        const uniqueVersions = new Set<string>();
+        return matches
+          .filter((match) => {
+            // Filter out prerelease and unsupported versions
+            const [_, major, minor, patch, changeSet] = match;
+            return patch.includes('f') && Number(major) >= 2017;
+          })
+          .map((match) => {
+            const [_, major, minor, patch, changeSet] = match;
+            const version = `${major}.${minor}.${patch}`;
+            if (!uniqueVersions.has(version)) {
+              uniqueVersions.add(version);
+              return {
+                version,
+                changeSet,
+                major: Number(major),
+                minor: Number(minor),
+                patch,
+              };
+            }
 
-    return {
-      version,
-      changeSet,
-      major: Number(major),
-      minor: Number(minor),
-      patch,
-    };
-  });
+            // Return null if version is not unique
+            return null;
+          })
+          .filter((version) => version !== null) as EditorVersionInfo[];
+      }
+    }
+  }
 
-  return versionInfoList;
+  throw new Error('No Unity versions found!');
 };