Configurable google rate limit / cooldown so we can test it

src/configuration.ts

@@ -42,7 +42,7 @@ const Config = t.strict({
   TURSO_TOKEN: t.union([t.undefined, t.string]),
   TURSO_SYNC_URL: t.union([t.undefined, t.string]),
   LOG_LEVEL: withDefaultIfEmpty(LogLevel, 'debug'),
-  QUIZ_RESULT_REFRESH_COOLDOWN_MS: withDefaultIfEmpty(
+  GOOGLE_RATELIMIT_MS: withDefaultIfEmpty(
     tt.IntFromString,
     (20 * 60 * 1000) as t.Int
   ),

src/init-dependencies/init-dependencies.ts

@@ -66,7 +66,8 @@ export const initDependencies = (
   const sharedReadModel = initSharedReadModel(
     dbClient,
     logger,
-    pullGoogleSheetData(googleAuth)
+    pullGoogleSheetData(googleAuth),
+    conf.GOOGLE_RATELIMIT_MS
   );
 
   const deps: Dependencies = {

src/read-models/shared-state/async-apply-external-event-sources.ts

@@ -13,8 +13,6 @@ import {QzEvent} from '../../types/qz-event';
 import {extractGoogleSheetData} from '../../training-sheets/google';
 import {UUID} from 'io-ts-types';
 
-const GOOGLE_UPDATE_INTERVAL_MS = 5 * 60 * 1000;
-
 export type PullSheetData = (
   logger: Logger,
   trainingSheetId: string
@@ -63,7 +61,8 @@ export const asyncApplyExternalEventSources = (
   logger: Logger,
   currentState: BetterSQLite3Database,
   pullGoogleSheetData: PullSheetData,
-  updateState: (event: DomainEvent) => void
+  updateState: (event: DomainEvent) => void,
+  googleRateLimitMs: number
 ) => {
   return () => async () => {
     logger.info('Applying external event sources...');
@@ -72,7 +71,7 @@ export const asyncApplyExternalEventSources = (
         O.isNone(equipment.lastQuizSync) ||
         (Date.now() as EpochTimestampMilliseconds) -
           equipment.lastQuizSync.value >
-          GOOGLE_UPDATE_INTERVAL_MS
+          googleRateLimitMs
       ) {
         logger.info(
           'Triggering event update from google training sheets for %s...',

src/read-models/shared-state/index.ts

@@ -33,7 +33,8 @@ export type SharedReadModel = {
 export const initSharedReadModel = (
   eventStoreClient: Client,
   logger: Logger,
-  pullGoogleSheetData: PullSheetData
+  pullGoogleSheetData: PullSheetData,
+  googleRateLimitMs: number
 ): SharedReadModel => {
   const readModelDb = drizzle(new Database());
   createTables.forEach(statement => readModelDb.run(statement));
@@ -46,7 +47,8 @@ export const initSharedReadModel = (
       logger,
       readModelDb,
       pullGoogleSheetData,
-      updateState_
+      updateState_,
+      googleRateLimitMs
     ),
     members: {
       get: getMember(readModelDb),

tests/init-dependencies/happy-path-adapters.helper.ts

@@ -19,7 +19,8 @@ export const happyPathAdapters: Dependencies = {
       level: 'fatal',
       timestamp: pino.stdTimeFunctions.isoTime,
     }),
-    localPullGoogleSheetData
+    localPullGoogleSheetData,
+    120_000
   ),
   logger: (() => undefined) as never as Logger,
   rateLimitSendingOfEmails: TE.right,

tests/read-models/test-framework.ts

@@ -54,7 +54,8 @@ export const initTestFramework = async (): Promise<TestFramework> => {
   const sharedReadModel = initSharedReadModel(
     dbClient,
     logger,
-    localPullGoogleSheetData
+    localPullGoogleSheetData,
+    120_000
   );
   const frameworkCommitEvent = commitEvent(
     dbClient,

tests/training-sheets/process-events.test.ts

@@ -16,7 +16,6 @@ import {
   EpochTimestampMilliseconds,
   Equipment,
 } from '../../src/read-models/shared-state/return-types';
-import {DateTime} from 'luxon';
 import {getSomeOrFail} from '../helpers';
 
 const sortQuizResults = RA.sort({
@@ -69,28 +68,31 @@ const extractEvents = async (
   return await pullNewEquipmentQuizResultsLocal(equipment);
 };
 
+type ApplyExternalEventsResults = {
+  startTime: EpochTimestampMilliseconds;
+  newEvents: DomainEvent[];
+  endTime: EpochTimestampMilliseconds;
+  equipmentAfter: Map<string, Equipment>;
+};
+
 const runAsyncApplyExternalEventSources = async (
   logger: Logger,
-  framework: TestFramework
-) => {
-  const startTime = DateTime.utc().toSeconds();
+  framework: TestFramework,
+  googleRateLimitMs: number
+): Promise<ApplyExternalEventsResults> => {
+  const startTime = Date.now() as EpochTimestampMilliseconds;
   const newEvents: DomainEvent[] = [];
   await asyncApplyExternalEventSources(
     logger,
     framework.sharedReadModel.db,
     localPullGoogleSheetData,
-    newEvents.push
+    newEvents.push,
+    googleRateLimitMs
   )()();
-  const endTime = DateTime.utc().toSeconds();
+  const endTime = Date.now() as EpochTimestampMilliseconds;
   const equipmentAfter = new Map(
     framework.sharedReadModel.equipment.getAll().map(e => [e.id, e])
   );
-  // Check that the last quiz sync property is updated to reflect
-  // that a quiz sync was preformed.
-  for (const equipment of equipmentAfter.values()) {
-    expect(equipment.lastQuizSync).toBeGreaterThan(startTime);
-    expect(equipment.lastQuizSync).toBeLessThan(endTime);
-  }
   return {
     startTime,
     newEvents,
@@ -99,6 +101,15 @@ const runAsyncApplyExternalEventSources = async (
   };
 };
 
+const checkLastQuizSync = (results: ApplyExternalEventsResults) => {
+  // Check that the last quiz sync property is updated to reflect
+  // that a quiz sync was preformed.
+  for (const equipment of results.equipmentAfter.values()) {
+    expect(equipment.lastQuizSync).toBeGreaterThan(results.startTime);
+    expect(equipment.lastQuizSync).toBeLessThan(results.endTime);
+  }
+};
+
 const checkLastQuizEventTimestamp = (
   data: gsheetData.ManualParsed,
   equipmentAfter: Equipment
@@ -258,18 +269,20 @@ describe('Training sheets worker', () => {
       const addWithSheet = async (
         name: string,
         areaId: UUID,
-        trainingSheetId: string
+        trainingSheetId: O.Option<string>
       ) => {
         const equipment = {
           id: faker.string.uuid() as UUID,
           name: name as NonEmptyString,
           areaId,
         };
         await framework.commands.equipment.add(equipment);
-        await framework.commands.equipment.trainingSheet({
-          equipmentId: equipment.id,
-          trainingSheetId,
-        });
+        if (O.isSome(trainingSheetId)) {
+          await framework.commands.equipment.trainingSheet({
+            equipmentId: equipment.id,
+            trainingSheetId: trainingSheetId.value,
+          });
+        }
         return {
           ...equipment,
           trainingSheetId,
@@ -280,17 +293,19 @@ describe('Training sheets worker', () => {
         const bambu = await addWithSheet(
           'bambu',
           createArea.id,
-          gsheetData.BAMBU.data.spreadsheetId!
+          O.some(gsheetData.BAMBU.data.spreadsheetId!)
         );
         const lathe = await addWithSheet(
           'Metal Lathe',
           createArea.id,
-          gsheetData.METAL_LATHE.data.spreadsheetId!
+          O.some(gsheetData.METAL_LATHE.data.spreadsheetId!)
         );
         const results = await runAsyncApplyExternalEventSources(
           logger,
-          framework
+          framework,
+          10_000
         );
+        checkLastQuizSync(results);
         checkLastQuizEventTimestamp(
           gsheetData.BAMBU,
           results.equipmentAfter.get(bambu.id)!
@@ -299,19 +314,82 @@ describe('Training sheets worker', () => {
           gsheetData.METAL_LATHE,
           results.equipmentAfter.get(lathe.id)!
         );
-        expect(results['newEvents']).toHaveLength(
+        expect(results.newEvents).toHaveLength(
           gsheetData.BAMBU.entries.length +
             gsheetData.METAL_LATHE.entries.length
         );
       });
       it('Handle no equipment', async () => {
         const results = await runAsyncApplyExternalEventSources(
           logger,
-          framework
+          framework,
+          10_000
         );
+        checkLastQuizSync(results);
         expect(results.equipmentAfter).toHaveLength(0);
       });
-      it('Rate limit equipment pull', () => {});
+      it('Handle equipment with no training sheet', async () => {
+        const bambu = await addWithSheet('bambu', createArea.id, O.none);
+        const results = await runAsyncApplyExternalEventSources(
+          logger,
+          framework,
+          10_000
+        );
+        checkLastQuizSync(results);
+        expect(
+          results.equipmentAfter.get(bambu.id)!.lastQuizResult
+        ).toStrictEqual(O.none);
+        expect(results.newEvents).toHaveLength(0);
+      });
+      it('Rate limit equipment pull', async () => {
+        const bambu = await addWithSheet(
+          'bambu',
+          createArea.id,
+          O.some(gsheetData.BAMBU.data.spreadsheetId!)
+        );
+        const results1 = await runAsyncApplyExternalEventSources(
+          logger,
+          framework,
+          10_000
+        );
+        checkLastQuizSync(results1);
+        const results2 = await runAsyncApplyExternalEventSources(
+          logger,
+          framework,
+          10_000
+        );
+        expect(
+          results1.equipmentAfter.get(bambu.id)!.lastQuizSync
+        ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync);
+        expect(results1.newEvents.length).toBeGreaterThan(0);
+        expect(results2.newEvents).toHaveLength(0);
+      });
+      it('Repeat equipment pull no rate limit', async () => {
+        const bambu = await addWithSheet(
+          'bambu',
+          createArea.id,
+          O.some(gsheetData.BAMBU.data.spreadsheetId!)
+        );
+        const results1 = await runAsyncApplyExternalEventSources(
+          logger,
+          framework,
+          100
+        );
+        checkLastQuizSync(results1);
+
+        await new Promise(res => setTimeout(res, 1000));
+        const results2 = await runAsyncApplyExternalEventSources(
+          logger,
+          framework,
+          100
+        );
+        checkLastQuizSync(results2);
+        expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual(
+          results2.equipmentAfter.get(bambu.id)!.lastQuizSync
+        );
+        expect(results1.newEvents.length).toBeGreaterThan(0);
+        expect(results2.newEvents).toHaveLength(0);
+      });
       it('Handle equipment in different areas', () => {});
     });
   });