From 638a203e466b4fd83d5f5f24bef9f830e398fee1 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Fri, 20 Sep 2024 22:48:01 +0100 Subject: [PATCH] Configurable google rate limit / cooldown so we can test it --- src/configuration.ts | 2 +- src/init-dependencies/init-dependencies.ts | 3 +- .../async-apply-external-event-sources.ts | 7 +- src/read-models/shared-state/index.ts | 6 +- .../happy-path-adapters.helper.ts | 3 +- tests/read-models/test-framework.ts | 3 +- tests/training-sheets/process-events.test.ts | 124 ++++++++++++++---- 7 files changed, 115 insertions(+), 33 deletions(-) diff --git a/src/configuration.ts b/src/configuration.ts index 09ffb48..b698285 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -42,7 +42,7 @@ const Config = t.strict({ TURSO_TOKEN: t.union([t.undefined, t.string]), TURSO_SYNC_URL: t.union([t.undefined, t.string]), LOG_LEVEL: withDefaultIfEmpty(LogLevel, 'debug'), - QUIZ_RESULT_REFRESH_COOLDOWN_MS: withDefaultIfEmpty( + GOOGLE_RATELIMIT_MS: withDefaultIfEmpty( tt.IntFromString, (20 * 60 * 1000) as t.Int ), diff --git a/src/init-dependencies/init-dependencies.ts b/src/init-dependencies/init-dependencies.ts index e81d007..cbbc72e 100644 --- a/src/init-dependencies/init-dependencies.ts +++ b/src/init-dependencies/init-dependencies.ts @@ -66,7 +66,8 @@ export const initDependencies = ( const sharedReadModel = initSharedReadModel( dbClient, logger, - pullGoogleSheetData(googleAuth) + pullGoogleSheetData(googleAuth), + conf.GOOGLE_RATELIMIT_MS ); const deps: Dependencies = { diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 7c10a66..a4691b2 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -13,8 +13,6 @@ import {QzEvent} from '../../types/qz-event'; import {extractGoogleSheetData} from '../../training-sheets/google'; import {UUID} from 'io-ts-types'; -const GOOGLE_UPDATE_INTERVAL_MS = 5 * 60 * 1000; - export type PullSheetData = ( logger: Logger, trainingSheetId: string @@ -63,7 +61,8 @@ export const asyncApplyExternalEventSources = ( logger: Logger, currentState: BetterSQLite3Database, pullGoogleSheetData: PullSheetData, - updateState: (event: DomainEvent) => void + updateState: (event: DomainEvent) => void, + googleRateLimitMs: number ) => { return () => async () => { logger.info('Applying external event sources...'); @@ -72,7 +71,7 @@ export const asyncApplyExternalEventSources = ( O.isNone(equipment.lastQuizSync) || (Date.now() as EpochTimestampMilliseconds) - equipment.lastQuizSync.value > - GOOGLE_UPDATE_INTERVAL_MS + googleRateLimitMs ) { logger.info( 'Triggering event update from google training sheets for %s...', diff --git a/src/read-models/shared-state/index.ts b/src/read-models/shared-state/index.ts index 561f4fb..5e86389 100644 --- a/src/read-models/shared-state/index.ts +++ b/src/read-models/shared-state/index.ts @@ -33,7 +33,8 @@ export type SharedReadModel = { export const initSharedReadModel = ( eventStoreClient: Client, logger: Logger, - pullGoogleSheetData: PullSheetData + pullGoogleSheetData: PullSheetData, + googleRateLimitMs: number ): SharedReadModel => { const readModelDb = drizzle(new Database()); createTables.forEach(statement => readModelDb.run(statement)); @@ -46,7 +47,8 @@ export const initSharedReadModel = ( logger, readModelDb, pullGoogleSheetData, - updateState_ + updateState_, + googleRateLimitMs ), members: { get: getMember(readModelDb), diff --git a/tests/init-dependencies/happy-path-adapters.helper.ts b/tests/init-dependencies/happy-path-adapters.helper.ts index 13c2a94..96e362d 100644 --- a/tests/init-dependencies/happy-path-adapters.helper.ts +++ b/tests/init-dependencies/happy-path-adapters.helper.ts @@ -19,7 +19,8 @@ export const happyPathAdapters: Dependencies = { level: 'fatal', timestamp: pino.stdTimeFunctions.isoTime, }), - localPullGoogleSheetData + localPullGoogleSheetData, + 120_000 ), logger: (() => undefined) as never as Logger, rateLimitSendingOfEmails: TE.right, diff --git a/tests/read-models/test-framework.ts b/tests/read-models/test-framework.ts index 9b02c2e..09d51f0 100644 --- a/tests/read-models/test-framework.ts +++ b/tests/read-models/test-framework.ts @@ -54,7 +54,8 @@ export const initTestFramework = async (): Promise => { const sharedReadModel = initSharedReadModel( dbClient, logger, - localPullGoogleSheetData + localPullGoogleSheetData, + 120_000 ); const frameworkCommitEvent = commitEvent( dbClient, diff --git a/tests/training-sheets/process-events.test.ts b/tests/training-sheets/process-events.test.ts index 8ec645b..2119210 100644 --- a/tests/training-sheets/process-events.test.ts +++ b/tests/training-sheets/process-events.test.ts @@ -16,7 +16,6 @@ import { EpochTimestampMilliseconds, Equipment, } from '../../src/read-models/shared-state/return-types'; -import {DateTime} from 'luxon'; import {getSomeOrFail} from '../helpers'; const sortQuizResults = RA.sort({ @@ -69,28 +68,31 @@ const extractEvents = async ( return await pullNewEquipmentQuizResultsLocal(equipment); }; +type ApplyExternalEventsResults = { + startTime: EpochTimestampMilliseconds; + newEvents: DomainEvent[]; + endTime: EpochTimestampMilliseconds; + equipmentAfter: Map; +}; + const runAsyncApplyExternalEventSources = async ( logger: Logger, - framework: TestFramework -) => { - const startTime = DateTime.utc().toSeconds(); + framework: TestFramework, + googleRateLimitMs: number +): Promise => { + const startTime = Date.now() as EpochTimestampMilliseconds; const newEvents: DomainEvent[] = []; await asyncApplyExternalEventSources( logger, framework.sharedReadModel.db, localPullGoogleSheetData, - newEvents.push + newEvents.push, + googleRateLimitMs )()(); - const endTime = DateTime.utc().toSeconds(); + const endTime = Date.now() as EpochTimestampMilliseconds; const equipmentAfter = new Map( framework.sharedReadModel.equipment.getAll().map(e => [e.id, e]) ); - // Check that the last quiz sync property is updated to reflect - // that a quiz sync was preformed. - for (const equipment of equipmentAfter.values()) { - expect(equipment.lastQuizSync).toBeGreaterThan(startTime); - expect(equipment.lastQuizSync).toBeLessThan(endTime); - } return { startTime, newEvents, @@ -99,6 +101,15 @@ const runAsyncApplyExternalEventSources = async ( }; }; +const checkLastQuizSync = (results: ApplyExternalEventsResults) => { + // Check that the last quiz sync property is updated to reflect + // that a quiz sync was preformed. + for (const equipment of results.equipmentAfter.values()) { + expect(equipment.lastQuizSync).toBeGreaterThan(results.startTime); + expect(equipment.lastQuizSync).toBeLessThan(results.endTime); + } +}; + const checkLastQuizEventTimestamp = ( data: gsheetData.ManualParsed, equipmentAfter: Equipment @@ -258,7 +269,7 @@ describe('Training sheets worker', () => { const addWithSheet = async ( name: string, areaId: UUID, - trainingSheetId: string + trainingSheetId: O.Option ) => { const equipment = { id: faker.string.uuid() as UUID, @@ -266,10 +277,12 @@ describe('Training sheets worker', () => { areaId, }; await framework.commands.equipment.add(equipment); - await framework.commands.equipment.trainingSheet({ - equipmentId: equipment.id, - trainingSheetId, - }); + if (O.isSome(trainingSheetId)) { + await framework.commands.equipment.trainingSheet({ + equipmentId: equipment.id, + trainingSheetId: trainingSheetId.value, + }); + } return { ...equipment, trainingSheetId, @@ -280,17 +293,19 @@ describe('Training sheets worker', () => { const bambu = await addWithSheet( 'bambu', createArea.id, - gsheetData.BAMBU.data.spreadsheetId! + O.some(gsheetData.BAMBU.data.spreadsheetId!) ); const lathe = await addWithSheet( 'Metal Lathe', createArea.id, - gsheetData.METAL_LATHE.data.spreadsheetId! + O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) ); const results = await runAsyncApplyExternalEventSources( logger, - framework + framework, + 10_000 ); + checkLastQuizSync(results); checkLastQuizEventTimestamp( gsheetData.BAMBU, results.equipmentAfter.get(bambu.id)! @@ -299,7 +314,7 @@ describe('Training sheets worker', () => { gsheetData.METAL_LATHE, results.equipmentAfter.get(lathe.id)! ); - expect(results['newEvents']).toHaveLength( + expect(results.newEvents).toHaveLength( gsheetData.BAMBU.entries.length + gsheetData.METAL_LATHE.entries.length ); @@ -307,11 +322,74 @@ describe('Training sheets worker', () => { it('Handle no equipment', async () => { const results = await runAsyncApplyExternalEventSources( logger, - framework + framework, + 10_000 ); + checkLastQuizSync(results); expect(results.equipmentAfter).toHaveLength(0); }); - it('Rate limit equipment pull', () => {}); + it('Handle equipment with no training sheet', async () => { + const bambu = await addWithSheet('bambu', createArea.id, O.none); + const results = await runAsyncApplyExternalEventSources( + logger, + framework, + 10_000 + ); + checkLastQuizSync(results); + expect( + results.equipmentAfter.get(bambu.id)!.lastQuizResult + ).toStrictEqual(O.none); + expect(results.newEvents).toHaveLength(0); + }); + it('Rate limit equipment pull', async () => { + const bambu = await addWithSheet( + 'bambu', + createArea.id, + O.some(gsheetData.BAMBU.data.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources( + logger, + framework, + 10_000 + ); + checkLastQuizSync(results1); + const results2 = await runAsyncApplyExternalEventSources( + logger, + framework, + 10_000 + ); + expect( + results1.equipmentAfter.get(bambu.id)!.lastQuizSync + ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync); + expect(results1.newEvents.length).toBeGreaterThan(0); + expect(results2.newEvents).toHaveLength(0); + }); + it('Repeat equipment pull no rate limit', async () => { + const bambu = await addWithSheet( + 'bambu', + createArea.id, + O.some(gsheetData.BAMBU.data.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources( + logger, + framework, + 100 + ); + checkLastQuizSync(results1); + + await new Promise(res => setTimeout(res, 1000)); + const results2 = await runAsyncApplyExternalEventSources( + logger, + framework, + 100 + ); + checkLastQuizSync(results2); + expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( + results2.equipmentAfter.get(bambu.id)!.lastQuizSync + ); + expect(results1.newEvents.length).toBeGreaterThan(0); + expect(results2.newEvents).toHaveLength(0); + }); it('Handle equipment in different areas', () => {}); }); });