From 31e7fd401f33679290f76473523c7a8b2630e207 Mon Sep 17 00:00:00 2001 From: Andrew Kahr <22359829+AndrewKahr@users.noreply.github.com> Date: Wed, 5 Jun 2024 08:01:15 -0700 Subject: [PATCH] Fix Unity Scrape Method (#51) * Improved deployment tests. More github auth fixes * More function deployment testing * Fix formatting * Force curl to print error info * Remove stray return * Test new scrape method. Add scrape test to testFunction * Only include f releases. Print all unity versions on response for test function * Fix index * Remove duplicate versions * Match original order * Filter out Unity 5 --- .github/workflows/main.yml | 6 +++ functions/src/api/testFunction.ts | 45 ++++++++++++++-- functions/src/config/settings.ts | 1 + .../ingestUnityVersions/scrapeVersions.ts | 52 ++++++++++++------- 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42281e3..b8d2160 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,8 +58,13 @@ jobs: args: deploy --only functions:testFunction env: GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' + + - name: Call Test Function + run: curl -f -s -S -X POST https://testfunction-wbe4ukn6tq-ey.a.run.app + - name: Cleanup Firebase Test uses: w9jds/firebase-action@v13.10.2 + if: always() with: args: functions:delete testFunction --force env: @@ -86,6 +91,7 @@ jobs: GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' - name: Cleanup Firebase Test uses: w9jds/firebase-action@v13.10.2 + if: always() with: args: functions:delete testFunction --force env: diff --git a/functions/src/api/testFunction.ts b/functions/src/api/testFunction.ts index 0a523f6..896865b 100644 --- a/functions/src/api/testFunction.ts +++ b/functions/src/api/testFunction.ts @@ -1,6 +1,10 @@ import { onRequest, Request } from 'firebase-functions/v2/https'; import { Response } from 'express-serve-static-core'; import { defineSecret } from 'firebase-functions/params'; +import { scrapeVersions } from '../logic/ingestRepoVersions/scrapeVersions'; +import { scrapeVersions as scrapeUnityVersions } from '../logic/ingestUnityVersions/scrapeVersions'; + +import { Discord } from '../service/discord'; const discordToken = defineSecret('DISCORD_TOKEN'); const githubPrivateKeyConfigSecret = defineSecret('GITHUB_PRIVATE_KEY'); @@ -9,7 +13,7 @@ const internalToken = defineSecret('INTERNAL_TOKEN'); export const testFunction = onRequest( { - // Passing secrets so that test deployments verify that the secrets are correctly set. + // Passing all secrets so that test deployments verify that the secrets are correctly set. secrets: [ discordToken, githubPrivateKeyConfigSecret, @@ -17,6 +21,41 @@ export const testFunction = onRequest( internalToken, ], }, - // eslint-disable-next-line @typescript-eslint/no-empty-function - async (request: Request, response: Response) => {}, + async (request: Request, response: Response) => { + // Run all non-sensitive functions to verify that the deployment is working. + const discordClient = new Discord(); + let info = 'Ok'; + let code = 200; + + try { + await discordClient.init(discordToken.value()); + + const versions = await scrapeVersions( + githubPrivateKeyConfigSecret.value(), + githubClientSecretConfigSecret.value(), + ); + + if (versions.length === 0) { + throw new Error('No versions were found.'); + } + + const unityVersions = await scrapeUnityVersions(); + if (unityVersions.length === 0) { + throw new Error('No Unity versions were found.'); + } + + info = `Found ${versions.length} repo versions and ${ + unityVersions.length + } Unity versions. Unity Versions: \n${unityVersions + .map((unity) => `${unity.version}:${unity.changeSet}`) + .join('\n')}`; + } catch (error: any) { + info = error.message; + code = 500; + } finally { + await discordClient.disconnect(); + } + + response.status(code).send(info); + }, ); diff --git a/functions/src/config/settings.ts b/functions/src/config/settings.ts index 4c98287..025aecf 100644 --- a/functions/src/config/settings.ts +++ b/functions/src/config/settings.ts @@ -21,6 +21,7 @@ export const settings = { github: { auth: { appId: 84327, + installationId: 12321333, clientId: 'Iv1.fa93dce6a47c9357', }, }, diff --git a/functions/src/logic/ingestUnityVersions/scrapeVersions.ts b/functions/src/logic/ingestUnityVersions/scrapeVersions.ts index 46c9cea..390cb1a 100644 --- a/functions/src/logic/ingestUnityVersions/scrapeVersions.ts +++ b/functions/src/logic/ingestUnityVersions/scrapeVersions.ts @@ -2,29 +2,45 @@ import { getDocumentFromUrl } from '../utils/get-document-from-url'; import { EditorVersionInfo } from '../../model/editorVersionInfo'; const UNITY_ARCHIVE_URL = 'https://unity.com/releases/editor/archive'; +const unity_version_regex = /unityhub:\/\/(\d+)\.(\d+)\.(\d+[a-zA-Z]\d+)\/(\w+)/g; -/** - * Based on https://github.com/BLaZeKiLL/unity-scraper - */ export const scrapeVersions = async (): Promise => { const document = await getDocumentFromUrl(UNITY_ARCHIVE_URL); - const links = Array.from(document.querySelectorAll('.release-links div:first-child a[href]')); - const hrefs = links.map((a) => a.getAttribute('href')) as string[]; + const scripts = document.querySelectorAll('script'); - const versionInfoList = hrefs.map((href) => { - const info = href.replace('unityhub://', ''); - const [version, changeSet] = info.split('/'); - const [major, minor, patch] = version.split('.'); + for (const script of scripts) { + if (script.textContent) { + const matches = [...script.textContent.matchAll(unity_version_regex)]; + if (matches.length > 0) { + const uniqueVersions = new Set(); + return matches + .filter((match) => { + // Filter out prerelease and unsupported versions + const [_, major, minor, patch, changeSet] = match; + return patch.includes('f') && Number(major) >= 2017; + }) + .map((match) => { + const [_, major, minor, patch, changeSet] = match; + const version = `${major}.${minor}.${patch}`; + if (!uniqueVersions.has(version)) { + uniqueVersions.add(version); + return { + version, + changeSet, + major: Number(major), + minor: Number(minor), + patch, + }; + } - return { - version, - changeSet, - major: Number(major), - minor: Number(minor), - patch, - }; - }); + // Return null if version is not unique + return null; + }) + .filter((version) => version !== null) as EditorVersionInfo[]; + } + } + } - return versionInfoList; + throw new Error('No Unity versions found!'); };