diff --git a/.sentryignore b/.sentryignore deleted file mode 100644 index 4b17e01fbb..0000000000 --- a/.sentryignore +++ /dev/null @@ -1,2 +0,0 @@ -.gitignore -node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ab3788590..f7acc24fbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,20 +4,25 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). -#### [v6.139.0](https://github.com/opengovsg/FormSG/compare/v6.139.0...v6.139.0) +#### [v6.140.0](https://github.com/opengovsg/FormSG/compare/v6.139.0...v6.140.0) -- fix: forbid stripe acc connection with pdf summary enabled [`#7570`](https://github.com/opengovsg/FormSG/pull/7570) -- fix: failing gh actions due to runner out of disk space [`#7565`](https://github.com/opengovsg/FormSG/pull/7565) +- refactor: worker pool [`#7553`](https://github.com/opengovsg/FormSG/pull/7553) +- chore(admin): mrf v1.1 content [`#7579`](https://github.com/opengovsg/FormSG/pull/7579) +- build: merge release v6.139.0 to develop [`#7575`](https://github.com/opengovsg/FormSG/pull/7575) +- docs: remove Sentry references and .sentryignore file [`#7573`](https://github.com/opengovsg/FormSG/pull/7573) +- build: release v6.139.0 [`#7564`](https://github.com/opengovsg/FormSG/pull/7564) #### [v6.139.0](https://github.com/opengovsg/FormSG/compare/v6.138.0...v6.139.0) -> 5 August 2024 +> 6 August 2024 +- fix: forbid stripe acc connection with pdf summary enabled [`#7570`](https://github.com/opengovsg/FormSG/pull/7570) +- fix: failing gh actions due to runner out of disk space [`#7565`](https://github.com/opengovsg/FormSG/pull/7565) - fix: enable pdf attachment for encrypt mode forms [`#7523`](https://github.com/opengovsg/FormSG/pull/7523) - build: merge release v6.138.0 to develop [`#7559`](https://github.com/opengovsg/FormSG/pull/7559) - chore: remove unused sgid toggle [`#7563`](https://github.com/opengovsg/FormSG/pull/7563) - build: release v6.138.0 [`#7558`](https://github.com/opengovsg/FormSG/pull/7558) -- chore: bump version to v6.139.0 [`711b328`](https://github.com/opengovsg/FormSG/commit/711b328e999091e5e3ed52968087efa6894385ca) +- chore: bump version to v6.139.0 [`90ff12e`](https://github.com/opengovsg/FormSG/commit/90ff12e2178e29bc351fe6ef7c4b054a647b4258) #### [v6.138.0](https://github.com/opengovsg/FormSG/compare/v6.137.0...v6.138.0) diff --git a/CREDITS.md b/CREDITS.md index 036dda33fe..db58fe967e 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -645,70 +645,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- - -## Project -@sentry/browser - -### Source -https://github.com/getsentry/sentry-javascript - -### License -MIT License - -Copyright (c) 2022 Functional Software, Inc. dba Sentry - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- - -## Project -@sentry/integrations - -### Source -https://github.com/getsentry/sentry-javascript - -### License -MIT License - -Copyright (c) 2022 Functional Software, Inc. dba Sentry - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - ------------------------------------------------------------------------------- ## Project diff --git a/docs/DEPLOYMENT_SETUP.md b/docs/DEPLOYMENT_SETUP.md index 2f192f64ce..721ad19b61 100644 --- a/docs/DEPLOYMENT_SETUP.md +++ b/docs/DEPLOYMENT_SETUP.md @@ -46,7 +46,6 @@ SMS Analytics and Monitoring -- Sentry.io - Google Analytics Spam protection @@ -270,16 +269,6 @@ Forms can be protected with [recaptcha](https://www.google.com/recaptcha/about/) | :------------------------- | ----------------------------- | | `REACT_APP_GA_TRACKING_ID` | Google Analytics tracking ID. | -#### Sentry.io - -Client-side error events are piped to [sentry.io](https://sentry.io/welcome/) for monitoring purposes. - -| Variable | Description | -| :------------------ | ----------------------------------------------------------------------------------------------------- | -| `CSP_REPORT_URI` | Reporting URL for Content Security Policy violdations. Can be configured to use a Sentry.io endpoint. | -| `SENTRY_CONFIG_URL` | Sentry.io URL for configuring the Raven SDK. | -| `CSP_REPORT_URI` | Reporting URL for Content Security Policy violdations. Can be configured to use a Sentry.io endpoint. | - #### SMS with Twilio The Mobile Number field supports form-fillers verifying their mobile numbers via a One-Time-Pin sent to their mobile phones. All messages are sent using [Twilio](https://www.twilio.com/) messaging APIs. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fa5e5135cb..64f102ab4e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "form-frontend", - "version": "6.139.0", + "version": "6.140.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "form-frontend", - "version": "6.139.0", + "version": "6.140.0", "hasInstallScript": true, "dependencies": { "@chakra-ui/react": "^1.8.6", diff --git a/frontend/package.json b/frontend/package.json index 89400cca46..7fb53507d2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "form-frontend", - "version": "6.139.0", + "version": "6.140.0", "homepage": ".", "private": true, "dependencies": { diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts index 13b745a086..68c54ba3bf 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts @@ -1,6 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useMutation, UseMutationOptions } from 'react-query' import { datadogLogs } from '@datadog/browser-logs' +import { useFeature } from '@growthbook/growthbook-react' import { waitForMs } from '~utils/waitForMs' @@ -14,7 +15,10 @@ import { } from '~features/analytics/AnalyticsService' import { useUser } from '~features/user/queries' -import { downloadResponseAttachment } from './utils/downloadCsv' +import { + downloadResponseAttachment, + downloadResponseAttachmentURL, +} from './utils/downloadCsv' import { EncryptedResponseCsvGenerator } from './utils/EncryptedResponseCsvGenerator' import { EncryptedResponsesStreamParams, @@ -55,6 +59,19 @@ interface UseDecryptionWorkersProps { > } +function timeout( + ms: number, + errorMessage = 'Operation timed out', +): Promise { + return new Promise((_, reject) => + setTimeout(() => reject(new Error(errorMessage)), ms), + ) +} + +function withTimeout(promise: Promise, ms: number): Promise { + return Promise.race([promise, timeout(ms)]) +} + const useDecryptionWorkers = ({ onProgress, mutateProps, @@ -65,6 +82,11 @@ const useDecryptionWorkers = ({ const { data: adminForm } = useAdminForm() const { user } = useUser() + const isDev = process.env.NODE_ENV === 'development' + + const fasterDownloadsFeature = useFeature('faster-downloads') + const fasterDownloads = fasterDownloadsFeature.on || isDev + useEffect(() => { return () => killWorkers(workers) }, [workers]) @@ -161,13 +183,16 @@ const useDecryptionWorkers = ({ // round-robin scheduling const { workerApi } = workerPool[receivedRecordCount % numWorkers] - const decryptResult = await workerApi.decryptIntoCsv({ - line: result.value, - secretKey, - downloadAttachments, - formId: adminForm._id, - hostOrigin: window.location.origin, - }) + const decryptResult = await workerApi.decryptIntoCsv( + { + line: result.value, + secretKey, + downloadAttachments, + formId: adminForm._id, + hostOrigin: window.location.origin, + }, + fasterDownloads, + ) progress += 1 onProgress(progress) @@ -348,11 +373,320 @@ const useDecryptionWorkers = ({ }) }) }, - [adminForm, onProgress, user?._id, workers], + [adminForm, onProgress, user?._id, workers, fasterDownloads], + ) + + const downloadEncryptedResponsesFaster = useCallback( + async ({ + responsesCount, + downloadAttachments, + secretKey, + endDate, + startDate, + }: DownloadEncryptedParams) => { + if (!adminForm || !responsesCount) { + return Promise.resolve({ + expectedCount: 0, + successCount: 0, + errorCount: 0, + }) + } + + console.log('Faster downloads is enabled ⚡') + + abortControllerRef.current.abort() + const freshAbortController = new AbortController() + abortControllerRef.current = freshAbortController + + if (workers.length) killWorkers(workers) + + const numWorkers = window.navigator.hardwareConcurrency || 4 + let errorCount = 0 + let unverifiedCount = 0 + let attachmentErrorCount = 0 + let unknownStatusCount = 0 + + const logMeta = { + action: 'downloadEncryptedReponses', + formId: adminForm._id, + formTitle: adminForm.title, + downloadAttachments: downloadAttachments, + num_workers: numWorkers, + expectedNumSubmissions: NUM_OF_METADATA_ROWS, + adminId: user?._id, + } + // Trigger analytics here before starting decryption worker + trackDownloadResponseStart(adminForm, numWorkers, NUM_OF_METADATA_ROWS) + datadogLogs.logger.info('Download response start', { + meta: { + ...logMeta, + }, + }) + + const workerPool: CleanableDecryptionWorkerApi[] = [] + const idleWorkers: number[] = [] + + for (let i = workerPool.length; i < numWorkers; i++) { + workerPool.push(makeWorkerApiAndCleanup()) + idleWorkers.push(i) + } + + setWorkers(workerPool) + + const csvGenerator = new EncryptedResponseCsvGenerator( + responsesCount, + NUM_OF_METADATA_ROWS, + ) + + const stream = await getEncryptedResponsesStream( + adminForm._id, + { downloadAttachments, endDate, startDate }, + freshAbortController, + ) + + const processTask = async (value: string, workerIdx: number) => { + const { workerApi } = workerPool[workerIdx] + + const decryptResult = await workerApi.decryptIntoCsv( + { + line: value, + secretKey, + downloadAttachments, + formId: adminForm._id, + hostOrigin: window.location.origin, + }, + fasterDownloads, + ) + + switch (decryptResult.status) { + case CsvRecordStatus.Ok: + try { + csvGenerator.addRecord(decryptResult.submissionData) + } catch (e) { + errorCount++ + console.error('Error in getResponseInstance', e) + } + + // It's fine to hog on to the worker here while waiting for the browser + // rate limit to pass. If decryption is fast, we would wait regardless. + // If decryption is slow, we won't hit rate limits. + if (downloadAttachments && decryptResult.downloadBlobURL) { + await downloadResponseAttachmentURL( + decryptResult.downloadBlobURL, + decryptResult.id, + ) + URL.revokeObjectURL(decryptResult.downloadBlobURL) + } + break + case CsvRecordStatus.Unknown: + unknownStatusCount++ + break + case CsvRecordStatus.Error: + errorCount++ + break + case CsvRecordStatus.AttachmentError: + errorCount++ + attachmentErrorCount++ + break + case CsvRecordStatus.Unverified: + unverifiedCount++ + break + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const _: never = decryptResult.status + throw new Error('Invalid decryptResult status encountered.') + } + } + return workerIdx + } + + const readAndQueueTask = async () => { + const reader = stream.getReader() + let progress = 0 + let pendingTasks: Promise[] = [] + + try { + while (progress < responsesCount) { + const { done, value } = await reader.read() + if (done) break + + progress += 1 + onProgress(progress) + + while (idleWorkers.length === 0) { + const finishedTasks: number[] = [] + for (let i = 0; i < pendingTasks.length; i++) { + try { + const freedWorkerIdx = await withTimeout(pendingTasks[i], 50) + idleWorkers.push(freedWorkerIdx) + finishedTasks.push(i) + } catch (e) { + if ( + e instanceof Error && + e.message === 'Operation timed out' + ) { + continue + } + console.error(`Error in task ${i}`, e) + } + } + pendingTasks = pendingTasks.filter( + (_, i) => !finishedTasks.includes(i), + ) + } + + const workerIdx = idleWorkers.shift()! + pendingTasks.push(processTask(value, workerIdx)) + } + await Promise.all(pendingTasks) + } catch (e) { + console.error('Error reading stream', e) + } finally { + reader.releaseLock() + } + } + + const downloadStartTime = performance.now() + + return new Promise((resolve, reject) => { + readAndQueueTask() + .catch((err) => { + if (!downloadStartTime) { + // No start time, means did not even start http request. + datadogLogs.logger.info('Download network failure', { + meta: { + ...logMeta, + error: { + message: err.message, + name: err.name, + stack: err.stack, + }, + }, + }) + + trackDownloadNetworkFailure(adminForm, err) + } else { + const downloadFailedTime = performance.now() + const timeDifference = downloadFailedTime - downloadStartTime + + datadogLogs.logger.info('Download response failure', { + meta: { + ...logMeta, + duration: timeDifference, + error: { + message: err.message, + name: err.name, + stack: err.stack, + }, + }, + }) + + trackDownloadResponseFailure( + adminForm, + numWorkers, + NUM_OF_METADATA_ROWS, + timeDifference, + err, + ) + + console.error( + 'Failed to download data, is there a network issue?', + err, + ) + killWorkers(workerPool) + reject(err) + } + }) + .finally(() => { + const checkComplete = () => { + // If all the records could not be decrypted + if (errorCount + unverifiedCount === responsesCount) { + const failureEndTime = performance.now() + // todo: check the timedifference redeclaration + const timeDifference = failureEndTime - downloadStartTime + + datadogLogs.logger.info('Partial decryption failure', { + meta: { + ...logMeta, + duration: timeDifference, + error_count: errorCount, + unverified_count: unverifiedCount, + attachment_error_count: attachmentErrorCount, + unknown_status_count: unknownStatusCount, + }, + }) + + trackPartialDecryptionFailure( + adminForm, + numWorkers, + csvGenerator.length(), + timeDifference, + errorCount, + attachmentErrorCount, + ) + + killWorkers(workerPool) + resolve({ + expectedCount: responsesCount, + successCount: csvGenerator.length(), + errorCount, + unverifiedCount, + }) + } else if ( + // All results have been decrypted + csvGenerator.length() + errorCount + unverifiedCount >= + responsesCount + ) { + killWorkers(workerPool) + // Generate first three rows of meta-data before download + csvGenerator.addMetaDataFromSubmission( + errorCount, + unverifiedCount, + ) + csvGenerator.downloadCsv( + `${adminForm.title}-${adminForm._id}.csv`, + ) + + const downloadEndTime = performance.now() + const timeDifference = downloadEndTime - downloadStartTime + + datadogLogs.logger.info('Download response success', { + meta: { + ...logMeta, + duration: timeDifference, + }, + }) + + trackDownloadResponseSuccess( + adminForm, + numWorkers, + NUM_OF_METADATA_ROWS, + timeDifference, + ) + + resolve({ + expectedCount: responsesCount, + successCount: csvGenerator.length(), + errorCount, + unverifiedCount, + }) + } else { + setTimeout(checkComplete, 100) + } + } + + checkComplete() + }) + }) + }, + [adminForm, onProgress, user?._id, workers, fasterDownloads], ) const handleExportCsvMutation = useMutation( - (params: DownloadEncryptedParams) => downloadEncryptedResponses(params), + (params: DownloadEncryptedParams) => + fasterDownloads + ? downloadEncryptedResponsesFaster(params) + : downloadEncryptedResponses(params), mutateProps, ) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts index 55c24ac92d..2d550aa81a 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts @@ -13,6 +13,7 @@ import { /** @class CsvRecord represents the CSV data to be passed back, along with helper functions */ export class CsvRecord { downloadBlob?: Blob + downloadBlobURL?: string submissionData?: DecryptedSubmissionData #statusMessage: string diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/downloadCsv.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/downloadCsv.ts index 6c072d35b4..52173d49df 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/downloadCsv.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/downloadCsv.ts @@ -6,3 +6,10 @@ export const downloadResponseAttachment = async ( ) => { return FileSaver.saveAs(blob, 'RefNo ' + submissionId + '.zip') } + +export const downloadResponseAttachmentURL = async ( + blobURL: string, + submissionId: string, +) => { + return FileSaver.saveAs(blobURL, 'RefNo ' + submissionId + '.zip') +} diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts index 9adac0b77e..fe13b573b6 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts @@ -66,7 +66,10 @@ function verifySignature( * main thread. * @param data The data to decrypt into a csvRecord. */ -async function decryptIntoCsv(data: LineData): Promise { +async function decryptIntoCsv( + data: LineData, + fasterDownloads: boolean, +): Promise { // This needs to be dynamically imported due to sharing code between main app and worker code. // Fixes issue raised at https://stackoverflow.com/questions/66472945/referenceerror-refreshreg-is-not-defined // Something to do with babel-loader. @@ -189,7 +192,11 @@ async function decryptIntoCsv(data: LineData): Promise { CsvRecordStatus.Ok, 'Success (with Downloaded Attachment)', ) - csvRecord.setDownloadBlob(downloadBlob) + if (fasterDownloads) { + csvRecord.downloadBlobURL = URL.createObjectURL(downloadBlob) + } else { + csvRecord.setDownloadBlob(downloadBlob) + } } catch (error) { csvRecord.setStatus( CsvRecordStatus.AttachmentError, diff --git a/frontend/src/features/workspace/components/CreateFormModal/CreateFormModalContent/FormResponseOptions.tsx b/frontend/src/features/workspace/components/CreateFormModal/CreateFormModalContent/FormResponseOptions.tsx index 5ac4f717b0..7484d85057 100644 --- a/frontend/src/features/workspace/components/CreateFormModal/CreateFormModalContent/FormResponseOptions.tsx +++ b/frontend/src/features/workspace/components/CreateFormModal/CreateFormModalContent/FormResponseOptions.tsx @@ -36,14 +36,16 @@ export const FormResponseOptions = forwardRef< Recommended} isActive={value === FormResponseMode.Encrypt} onClick={() => onChange(FormResponseMode.Encrypt)} isFullWidth flex={1} > Storage mode form - View or download responses in FormSG + + View and download responses in FormSG or receive responses in your + inbox + Email mode form - Receive responses in your inbox + Receive responses in your inbox only Multi-respondent form - Create a workflow to collect responses from multiple respondents in - the same form submission + Collect responses from multiple people in the same submission diff --git a/package-lock.json b/package-lock.json index 41ede2255d..57b87b0c42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "FormSG", - "version": "6.139.0", + "version": "6.140.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "FormSG", - "version": "6.139.0", + "version": "6.140.0", "hasInstallScript": true, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "^3.536.0", diff --git a/package.json b/package.json index 910589dd4a..77023f1756 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "6.139.0", + "version": "6.140.0", "homepage": "https://form.gov.sg", "authors": [ "FormSG "