From aaf88aa30c986790798aea1bdcd62a8c548b17e5 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 24 Jan 2024 23:29:39 +0100 Subject: [PATCH 1/4] #18 feat(app): call callbackurl with result --- app/.env.local.template | 3 +- app/package-lock.json | 240 +++++++++--------- app/src/app/api/download/[slug]/route.ts | 62 +++++ app/temporal/src/activities.ts | 53 +++- .../src/workflows/importer.workflow.ts | 70 ++++- 5 files changed, 291 insertions(+), 137 deletions(-) create mode 100644 app/src/app/api/download/[slug]/route.ts diff --git a/app/.env.local.template b/app/.env.local.template index 80e6f6d..66e17db 100644 --- a/app/.env.local.template +++ b/app/.env.local.template @@ -1,3 +1,4 @@ MINIO_HOST=localhost MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= \ No newline at end of file +MINIO_SECRET_KEY= +API_URL=http://localhost:3000 \ No newline at end of file diff --git a/app/package-lock.json b/app/package-lock.json index 7dd57de..086316a 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1393,6 +1393,126 @@ "node": ">= 10" } }, + "node_modules/@next/swc-darwin-x64": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz", + "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz", + "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz", + "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz", + "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz", + "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz", + "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz", + "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz", + "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -10208,126 +10328,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz", - "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz", - "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz", - "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz", - "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz", - "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz", - "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz", - "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz", - "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } } } } diff --git a/app/src/app/api/download/[slug]/route.ts b/app/src/app/api/download/[slug]/route.ts new file mode 100644 index 0000000..7206692 --- /dev/null +++ b/app/src/app/api/download/[slug]/route.ts @@ -0,0 +1,62 @@ +import * as env from "env-var"; +import * as Minio from "minio"; +import { NextRequest } from "next/server"; + +const minioClient = new Minio.Client({ + endPoint: env.get("MINIO_HOST").required().asString(), + port: 9000, + useSSL: false, + accessKey: env.get("MINIO_ACCESS_KEY").required().asString(), + secretKey: env.get("MINIO_SECRET_KEY").required().asString(), +}); + +/* + TODO: refactor this + This method is a huge security risk. + It allows anyone to download any file from any bucket. +*/ +export async function GET( + req: NextRequest, + { params }: { params: { slug: string } } +) { + const { slug: importerId } = params; + const fileReference = req.nextUrl.searchParams.get("fileReference"); + + if (!importerId) { + return new Response("importerId missing", { status: 500 }); + } + if (!fileReference) { + return new Response("file missing", { status: 500 }); + } + + const bucket = importerId; + const bucketExists = await minioClient.bucketExists(bucket); + + if (bucketExists === false) { + return new Response("bucket does not exist", { status: 500 }); + } + + const fileBuffer = await getFile(bucket, fileReference); + + return new Response(fileBuffer, { status: 200 }); +} + +async function getFile(bucket: string, fileReference: string): Promise { + // TODO: this throws S3Error - not found + try { + const fileStats = await minioClient.statObject(bucket, fileReference); + if (!fileStats) { + throw new Error(); + } + } catch (error) { + throw new Error(`could not find file ${fileReference} in bucket ${bucket}`); + } + const stream = await minioClient.getObject(bucket, fileReference); + const data = await new Promise((resolve, reject) => { + let chunks: Buffer[] = []; + stream.on("error", reject); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("data", (chunk) => chunks.push(chunk)); + }); + return data; +} diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts index a38d8c2..1d9e186 100644 --- a/app/temporal/src/activities.ts +++ b/app/temporal/src/activities.ts @@ -4,6 +4,7 @@ import { chunk } from "lodash"; import XLSX from "xlsx"; import { ColumnConfig } from "./domain/ColumnConfig"; import { DataAnalyzer, DataMappingRecommendation } from "./domain/DataAnalyzer"; +import { DataSetPatch } from "./domain/DataSet"; import { ValidatorType } from "./domain/validators"; import { FileStore } from "./infrastructure/FileStore"; import { Mapping } from "./workflows/importer.workflow"; @@ -149,8 +150,8 @@ export function makeActivities( fileReference: string; statsFileReference: string; validatorColumns: ValidatorColumns; - }) => { - const referenceId = params.fileReference.split("-")[1].split(".")[0]; + outputFileReference: string; + }): Promise<{ errorFileReference: string; errorCount: number }> => { const fileData = await fileStore.getFile( params.bucket, params.fileReference @@ -167,13 +168,15 @@ export function makeActivities( params.validatorColumns, statsData ); - const errorFileReference = `errors-${referenceId}.json`; await fileStore.putFile( params.bucket, - errorFileReference, + params.outputFileReference, Buffer.from(JSON.stringify(errorData)) ); - return errorFileReference; + return { + errorFileReference: params.outputFileReference, + errorCount: errorData.length, + }; }, generateStatsFile: async (params: { bucket: string; @@ -212,5 +215,45 @@ export function makeActivities( Buffer.from(JSON.stringify(allJsonData)) ); }, + applyPatches: async (params: { + bucket: string; + fileReference: string; + outputFileReference: string; + patches: DataSetPatch[]; + }) => { + const fileData = await fileStore.getFile( + params.bucket, + params.fileReference + ); + const allJsonData: Record[] = JSON.parse( + fileData.toString() + ); + + for (const patch of params.patches) { + const indexToUpdate = allJsonData.findIndex( + (item) => item.__rowId === patch.row + ); + allJsonData[indexToUpdate][patch.col] = patch.newValue; + } + + await fileStore.putFile( + params.bucket, + params.outputFileReference, + Buffer.from(JSON.stringify(allJsonData)) + ); + }, + export: async (params: { + bucket: string; + fileReference: string; + callbackUrl: string; + }) => { + const host = process.env.API_URL ?? "http://localhost:3000"; + const downloadUrl = `${host}/api/download/${params.bucket}/?fileReference=${params.fileReference}`; + console.log("downloadUrl", downloadUrl); + fetch(params.callbackUrl, { + method: "POST", + body: downloadUrl, + }); + }, }; } diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts index 46803a5..2e2ba38 100644 --- a/app/temporal/src/workflows/importer.workflow.ts +++ b/app/temporal/src/workflows/importer.workflow.ts @@ -7,6 +7,7 @@ import { proxyActivities, setHandler, } from "@temporalio/workflow"; +import { sum } from "lodash"; import pLimit from "p-limit"; import { ValidatorColumns, makeActivities } from "../activities"; import { ColumnConfig } from "../domain/ColumnConfig"; @@ -88,9 +89,10 @@ export async function importer(params: ImporterWorkflowParams) { let patches: DataSetPatch[] = []; let importStartRequested = false; let dataMappingRecommendations: DataMappingRecommendation[] | null = null; - let validationFileReferences: string[] | null = null; + let exportFileReference: string | null = null; let isValidating = false; let configuredMappings: Mapping[] | null = null; + let errorCount = 0; setHandler( addFileUpdate, @@ -133,9 +135,10 @@ export async function importer(params: ImporterWorkflowParams) { isWaitingForMapping: configuredMappings === null, isWaitingForImport: importStartRequested === false, isImporting: importStartRequested === true, + fileHasErrors: errorCount > 0, dataMappingRecommendations, isValidating, - validationFileReferences, + exportFileReference, dataMapping: configuredMappings, }; }); @@ -189,6 +192,26 @@ export async function importer(params: ImporterWorkflowParams) { "Timeout: import start not requested" ); } + + const hasValidationNoErrors = await condition( + () => errorCount === 0, + startImportTimeout // own timeout for errors? + ); + if (!hasValidationNoErrors) { + throw ApplicationFailure.nonRetryable( + "Timeout: validation still has errors" + ); + } + + // we dont have any more errors and the patched file includes all patches + exportFileReference = "patched.json"; + + // we dont need to await the callbackUrl call + void acts.export({ + bucket: sourceFile!.bucket, + fileReference: exportFileReference, + callbackUrl: params.callbackUrl, + }); } catch (err) { for (const compensation of compensations) { await compensation(); @@ -224,13 +247,23 @@ export async function importer(params: ImporterWorkflowParams) { } } } + // TODO: we are applying patches twice here, once for the whole file and once for each chunk + const patchedFileReference = "patched.json"; + if (patches.length > 0) { + await acts.applyPatches({ + bucket: sourceFile!.bucket, + fileReference: sourceFileReference, + outputFileReference: patchedFileReference, + patches, + }); + } - // TODO: apply patches const statsFileReference = "stats.json"; const startStats = Date.now(); await acts.generateStatsFile({ bucket: sourceFile!.bucket, - fileReference: sourceFileReference, + fileReference: + patches.length > 0 ? patchedFileReference : sourceFileReference, outputFileReference: statsFileReference, uniqueColumns: validatorColumns.unique.map((item) => item.column), }); @@ -239,19 +272,34 @@ export async function importer(params: ImporterWorkflowParams) { const startAllValidations = Date.now(); const limit = pLimit(100); const parallelValidations = chunkedFileReferences.map((fileReference) => - limit(() => - acts.processDataValidations({ + limit(async () => { + const referenceId = fileReference.split("-")[1].split(".")[0]; + // TODO: we are applying patches twice here, once for the whole file and once for each chunk + const patchedFileReference = `patched-${referenceId}.json`; + if (patches.length > 0) { + await acts.applyPatches({ + bucket: sourceFile!.bucket, + fileReference: sourceFileReference, + outputFileReference: patchedFileReference, + patches, + }); + } + const errorFileReference = `errors-${referenceId}.json`; + return acts.processDataValidations({ bucket: sourceFile!.bucket, - fileReference, + fileReference: + patches.length > 0 ? patchedFileReference : fileReference, statsFileReference, validatorColumns, - }) - ) + outputFileReference: errorFileReference, + }); + }) ); - const errorFileReferences = await Promise.all(parallelValidations); + const validationErrors = await Promise.all(parallelValidations); + errorCount = sum(validationErrors.map((item) => item.errorCount)); await acts.mergeChunks({ bucket: sourceFile!.bucket, - fileReferences: errorFileReferences, + fileReferences: validationErrors.map((item) => item.errorFileReference), outputFileReference: "errors.json", }); await acts.mergeChunks({ From 2626fb7245cf68664444281ecac85b35e2847e0d Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 25 Jan 2024 10:12:14 +0100 Subject: [PATCH 2/4] refactor(app): dont save stats to file / apply patches --- app/src/app/api/download/[slug]/route.ts | 11 +-- app/temporal/src/activities.ts | 93 ++++++++----------- app/temporal/src/domain/DataSet.ts | 2 +- .../src/workflows/importer.workflow.ts | 47 +++------- 4 files changed, 52 insertions(+), 101 deletions(-) diff --git a/app/src/app/api/download/[slug]/route.ts b/app/src/app/api/download/[slug]/route.ts index 7206692..4a1f1bb 100644 --- a/app/src/app/api/download/[slug]/route.ts +++ b/app/src/app/api/download/[slug]/route.ts @@ -10,24 +10,16 @@ const minioClient = new Minio.Client({ secretKey: env.get("MINIO_SECRET_KEY").required().asString(), }); -/* - TODO: refactor this - This method is a huge security risk. - It allows anyone to download any file from any bucket. -*/ export async function GET( req: NextRequest, { params }: { params: { slug: string } } ) { const { slug: importerId } = params; - const fileReference = req.nextUrl.searchParams.get("fileReference"); + const fileReference = "target.json"; if (!importerId) { return new Response("importerId missing", { status: 500 }); } - if (!fileReference) { - return new Response("file missing", { status: 500 }); - } const bucket = importerId; const bucketExists = await minioClient.bucketExists(bucket); @@ -42,7 +34,6 @@ export async function GET( } async function getFile(bucket: string, fileReference: string): Promise { - // TODO: this throws S3Error - not found try { const fileStats = await minioClient.statObject(bucket, fileReference); if (!fileStats) { diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts index 1d9e186..6abc98b 100644 --- a/app/temporal/src/activities.ts +++ b/app/temporal/src/activities.ts @@ -3,7 +3,11 @@ import csv from "csv"; import { chunk } from "lodash"; import XLSX from "xlsx"; import { ColumnConfig } from "./domain/ColumnConfig"; -import { DataAnalyzer, DataMappingRecommendation } from "./domain/DataAnalyzer"; +import { + DataAnalyzer, + DataMappingRecommendation, + Stats, +} from "./domain/DataAnalyzer"; import { DataSetPatch } from "./domain/DataSet"; import { ValidatorType } from "./domain/validators"; import { FileStore } from "./infrastructure/FileStore"; @@ -148,25 +152,21 @@ export function makeActivities( processDataValidations: async (params: { bucket: string; fileReference: string; - statsFileReference: string; validatorColumns: ValidatorColumns; outputFileReference: string; + stats: Stats; + patches: DataSetPatch[]; }): Promise<{ errorFileReference: string; errorCount: number }> => { const fileData = await fileStore.getFile( params.bucket, params.fileReference ); const jsonData = JSON.parse(fileData.toString()); - - const statsFileData = await fileStore.getFile( - params.bucket, - params.statsFileReference - ); - const statsData = JSON.parse(statsFileData.toString()); + const patchedData = applyPatches(jsonData, params.patches); const errorData = dataAnalyzer.processDataValidations( - jsonData, + patchedData, params.validatorColumns, - statsData + params.stats ); await fileStore.putFile( params.bucket, @@ -178,12 +178,12 @@ export function makeActivities( errorCount: errorData.length, }; }, - generateStatsFile: async (params: { + generateStats: async (params: { bucket: string; fileReference: string; - outputFileReference: string; uniqueColumns: string[]; - }): Promise => { + patches: DataSetPatch[]; + }): Promise => { const fileData = await fileStore.getFile( params.bucket, params.fileReference @@ -191,64 +191,30 @@ export function makeActivities( const jsonData: Record[] = JSON.parse( fileData.toString() ); - const stats = dataAnalyzer.getStats(jsonData, params.uniqueColumns); - const statsData = Buffer.from(JSON.stringify(stats)); - await fileStore.putFile( - params.bucket, - params.outputFileReference, - statsData - ); + const patchedData = applyPatches(jsonData, params.patches); + return dataAnalyzer.getStats(patchedData, params.uniqueColumns); }, mergeChunks: async (params: { bucket: string; fileReferences: string[]; outputFileReference: string; + patches?: DataSetPatch[]; }) => { let allJsonData: Record[] = []; for (const fileReference of params.fileReferences) { const fileData = await fileStore.getFile(params.bucket, fileReference); allJsonData.push(...JSON.parse(fileData.toString())); } + const patchedData = applyPatches(allJsonData, params.patches ?? []); await fileStore.putFile( params.bucket, params.outputFileReference, - Buffer.from(JSON.stringify(allJsonData)) - ); - }, - applyPatches: async (params: { - bucket: string; - fileReference: string; - outputFileReference: string; - patches: DataSetPatch[]; - }) => { - const fileData = await fileStore.getFile( - params.bucket, - params.fileReference - ); - const allJsonData: Record[] = JSON.parse( - fileData.toString() - ); - - for (const patch of params.patches) { - const indexToUpdate = allJsonData.findIndex( - (item) => item.__rowId === patch.row - ); - allJsonData[indexToUpdate][patch.col] = patch.newValue; - } - - await fileStore.putFile( - params.bucket, - params.outputFileReference, - Buffer.from(JSON.stringify(allJsonData)) + Buffer.from(JSON.stringify(patchedData)) ); }, - export: async (params: { - bucket: string; - fileReference: string; - callbackUrl: string; - }) => { + invokeCallback: async (params: { bucket: string; callbackUrl: string }) => { const host = process.env.API_URL ?? "http://localhost:3000"; - const downloadUrl = `${host}/api/download/${params.bucket}/?fileReference=${params.fileReference}`; + const downloadUrl = `${host}/api/download/${params.bucket}`; console.log("downloadUrl", downloadUrl); fetch(params.callbackUrl, { method: "POST", @@ -257,3 +223,22 @@ export function makeActivities( }, }; } + +function applyPatches( + data: Record[], + patches: DataSetPatch[] +): Record[] { + const newData = data.slice(); + for (const patch of patches) { + const indexToUpdate = newData.findIndex( + (item) => item.__rowId === patch.row + ); + if (indexToUpdate !== -1) { + newData[indexToUpdate] = { + ...newData[indexToUpdate], + [patch.col]: patch.newValue, + }; + } + } + return newData; +} diff --git a/app/temporal/src/domain/DataSet.ts b/app/temporal/src/domain/DataSet.ts index 1263d07..c53a524 100644 --- a/app/temporal/src/domain/DataSet.ts +++ b/app/temporal/src/domain/DataSet.ts @@ -1,5 +1,5 @@ export interface DataSetPatch { row: number; - col: number; + col: string; newValue: string | number | null; } diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts index 2e2ba38..e260e62 100644 --- a/app/temporal/src/workflows/importer.workflow.ts +++ b/app/temporal/src/workflows/importer.workflow.ts @@ -203,13 +203,11 @@ export async function importer(params: ImporterWorkflowParams) { ); } - // we dont have any more errors and the patched file includes all patches - exportFileReference = "patched.json"; + // we dont have any more errors and target.json has all patches applied + exportFileReference = "target.json"; - // we dont need to await the callbackUrl call - void acts.export({ + void acts.invokeCallback({ bucket: sourceFile!.bucket, - fileReference: exportFileReference, callbackUrl: params.callbackUrl, }); } catch (err) { @@ -247,51 +245,27 @@ export async function importer(params: ImporterWorkflowParams) { } } } - // TODO: we are applying patches twice here, once for the whole file and once for each chunk - const patchedFileReference = "patched.json"; - if (patches.length > 0) { - await acts.applyPatches({ - bucket: sourceFile!.bucket, - fileReference: sourceFileReference, - outputFileReference: patchedFileReference, - patches, - }); - } - const statsFileReference = "stats.json"; - const startStats = Date.now(); - await acts.generateStatsFile({ + const stats = await acts.generateStats({ bucket: sourceFile!.bucket, - fileReference: - patches.length > 0 ? patchedFileReference : sourceFileReference, - outputFileReference: statsFileReference, + fileReference: sourceFileReference, uniqueColumns: validatorColumns.unique.map((item) => item.column), + patches, }); - console.log(`generate stats file took ${Date.now() - startStats}ms`); const startAllValidations = Date.now(); const limit = pLimit(100); const parallelValidations = chunkedFileReferences.map((fileReference) => - limit(async () => { + limit(() => { const referenceId = fileReference.split("-")[1].split(".")[0]; - // TODO: we are applying patches twice here, once for the whole file and once for each chunk - const patchedFileReference = `patched-${referenceId}.json`; - if (patches.length > 0) { - await acts.applyPatches({ - bucket: sourceFile!.bucket, - fileReference: sourceFileReference, - outputFileReference: patchedFileReference, - patches, - }); - } const errorFileReference = `errors-${referenceId}.json`; return acts.processDataValidations({ bucket: sourceFile!.bucket, - fileReference: - patches.length > 0 ? patchedFileReference : fileReference, - statsFileReference, + fileReference, validatorColumns, outputFileReference: errorFileReference, + stats, + patches, }); }) ); @@ -306,6 +280,7 @@ export async function importer(params: ImporterWorkflowParams) { bucket: sourceFile!.bucket, fileReferences: chunkedFileReferences, outputFileReference: "target.json", + patches, }); console.log(`all validations took ${Date.now() - startAllValidations}ms`); isValidating = false; From d8437d3d743c43f72d4906013f86683397b9e89d Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 25 Jan 2024 10:45:09 +0100 Subject: [PATCH 3/4] chore(app): use minioClient --- app/package-lock.json | 164 +++++++---------------- app/src/app/api/download/[slug]/route.ts | 10 +- app/src/app/api/upload/route.ts | 10 +- 3 files changed, 52 insertions(+), 132 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 4718c78..694035c 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -67,7 +67,6 @@ }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -877,7 +876,6 @@ }, "node_modules/@isaacs/cliui": { "version": "8.0.2", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^5.1.2", @@ -893,7 +891,6 @@ }, "node_modules/@isaacs/cliui/node_modules/ansi-regex": { "version": "6.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -904,7 +901,6 @@ }, "node_modules/@isaacs/cliui/node_modules/strip-ansi": { "version": "7.1.0", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" @@ -1522,7 +1518,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -1535,7 +1530,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "engines": { "node": ">= 8" } @@ -1544,7 +1538,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -1557,7 +1550,6 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, "optional": true, "engines": { "node": ">=14" @@ -2588,13 +2580,13 @@ "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", - "dev": true + "devOptional": true }, "node_modules/@types/react": { "version": "18.2.48", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", - "dev": true, + "devOptional": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -2605,7 +2597,7 @@ "version": "18.2.18", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.18.tgz", "integrity": "sha512-TJxDm6OfAX2KJWJdMEVTwWke5Sc/E/RlnPGvGfS0W7+6ocy2xhDVQVh/KvC2Uf7kACs+gDytdusDSdWfWkaNzw==", - "dev": true, + "devOptional": true, "dependencies": { "@types/react": "*" } @@ -2614,7 +2606,7 @@ "version": "0.16.8", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==", - "dev": true + "devOptional": true }, "node_modules/@types/stack-utils": { "version": "2.0.3", @@ -3045,14 +3037,12 @@ "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -3427,14 +3417,12 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, "engines": { "node": ">=8" } @@ -3461,7 +3449,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -3585,7 +3572,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", - "dev": true, "engines": { "node": ">= 6" } @@ -3658,7 +3644,6 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, "funding": [ { "type": "individual", @@ -3685,7 +3670,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -3845,7 +3829,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, "engines": { "node": ">= 6" } @@ -3898,7 +3881,6 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -3912,7 +3894,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", - "dev": true, "bin": { "cssesc": "bin/cssesc" }, @@ -3924,7 +3905,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "dev": true + "devOptional": true }, "node_modules/csv": { "version": "6.3.6", @@ -4071,8 +4052,7 @@ "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", - "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", - "dev": true + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==" }, "node_modules/diff-sequences": { "version": "29.6.3", @@ -4098,8 +4078,7 @@ "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", - "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", - "dev": true + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==" }, "node_modules/doctrine": { "version": "3.0.0", @@ -4127,8 +4106,7 @@ "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "node_modules/electron-to-chromium": { "version": "1.4.639", @@ -4150,8 +4128,7 @@ "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, "node_modules/enhanced-resolve": { "version": "5.15.0", @@ -4818,11 +4795,16 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "peer": true + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -4838,7 +4820,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -4882,7 +4863,6 @@ "version": "1.16.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.16.0.tgz", "integrity": "sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==", - "dev": true, "dependencies": { "reusify": "^1.0.4" } @@ -4912,7 +4892,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4976,7 +4955,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dev": true, "dependencies": { "cross-spawn": "^7.0.0", "signal-exit": "^4.0.1" @@ -5024,7 +5002,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -5169,7 +5146,6 @@ "version": "10.3.10", "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", - "dev": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^2.3.5", @@ -5191,7 +5167,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, "dependencies": { "is-glob": "^4.0.3" }, @@ -5208,7 +5183,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -5217,7 +5191,6 @@ "version": "9.0.3", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", - "dev": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -5602,7 +5575,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -5641,7 +5613,6 @@ "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", - "dev": true, "dependencies": { "hasown": "^2.0.0" }, @@ -5668,7 +5639,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -5720,7 +5690,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -5753,7 +5722,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -5918,8 +5886,7 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", @@ -6013,7 +5980,6 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dev": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -6729,7 +6695,6 @@ "version": "1.21.0", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz", "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==", - "dev": true, "bin": { "jiti": "bin/jiti.js" } @@ -6919,7 +6884,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", - "dev": true, "engines": { "node": ">=10" } @@ -6927,8 +6891,7 @@ "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, "node_modules/load-json-file": { "version": "4.0.0", @@ -6987,6 +6950,18 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", + "peer": true + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "peer": true + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -7019,7 +6994,6 @@ "version": "10.1.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==", - "dev": true, "engines": { "node": "14 || >=16.14" } @@ -7099,7 +7073,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "engines": { "node": ">= 8" } @@ -7108,7 +7081,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -7194,7 +7166,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", - "dev": true, "engines": { "node": ">=16 || 14 >=14.17" } @@ -7209,7 +7180,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, "dependencies": { "any-promise": "^1.0.0", "object-assign": "^4.0.1", @@ -7422,7 +7392,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7615,7 +7584,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7624,7 +7592,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true, "engines": { "node": ">= 6" } @@ -7864,7 +7831,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "engines": { "node": ">=8" } @@ -7872,14 +7838,12 @@ "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-scurry": { "version": "1.10.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==", - "dev": true, "dependencies": { "lru-cache": "^9.1.1 || ^10.0.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -7909,7 +7873,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -7933,7 +7896,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7942,7 +7904,6 @@ "version": "4.0.6", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", - "dev": true, "engines": { "node": ">= 6" } @@ -8015,7 +7976,6 @@ "version": "8.4.33", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz", "integrity": "sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==", - "dev": true, "funding": [ { "type": "opencollective", @@ -8043,7 +8003,6 @@ "version": "15.1.0", "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", - "dev": true, "dependencies": { "postcss-value-parser": "^4.0.0", "read-cache": "^1.0.0", @@ -8060,7 +8019,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz", "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==", - "dev": true, "dependencies": { "camelcase-css": "^2.0.1" }, @@ -8079,7 +8037,6 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.2.tgz", "integrity": "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==", - "dev": true, "funding": [ { "type": "opencollective", @@ -8114,7 +8071,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.0.0.tgz", "integrity": "sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==", - "dev": true, "engines": { "node": ">=14" } @@ -8123,7 +8079,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.1.tgz", "integrity": "sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==", - "dev": true, "dependencies": { "postcss-selector-parser": "^6.0.11" }, @@ -8142,7 +8097,6 @@ "version": "6.0.15", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.15.tgz", "integrity": "sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==", - "dev": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -8154,8 +8108,7 @@ "node_modules/postcss-value-parser": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" }, "node_modules/prelude-ls": { "version": "1.2.1", @@ -8307,7 +8260,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -8323,6 +8275,20 @@ } ] }, + "node_modules/quill-delta": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", + "peer": true, + "dependencies": { + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -8431,7 +8397,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - "dev": true, "dependencies": { "pify": "^2.3.0" } @@ -8488,7 +8453,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -8550,7 +8514,6 @@ "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", - "dev": true, "dependencies": { "is-core-module": "^2.13.0", "path-parse": "^1.0.7", @@ -8615,7 +8578,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" @@ -8660,7 +8622,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -8844,7 +8805,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -8856,7 +8816,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "engines": { "node": ">=8" } @@ -8888,7 +8847,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, "engines": { "node": ">=14" }, @@ -9099,7 +9057,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -9117,7 +9074,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -9130,14 +9086,12 @@ "node_modules/string-width-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/string-width/node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true, "engines": { "node": ">=12" }, @@ -9149,7 +9103,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -9258,7 +9211,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -9327,7 +9279,6 @@ "version": "3.35.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==", - "dev": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", "commander": "^4.0.0", @@ -9361,7 +9312,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -9406,7 +9356,6 @@ "version": "3.4.1", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.1.tgz", "integrity": "sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==", - "dev": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -9554,7 +9503,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, "dependencies": { "any-promise": "^1.0.0" } @@ -9563,7 +9511,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, "dependencies": { "thenify": ">= 3.1.0 < 4" }, @@ -9609,7 +9556,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "dependencies": { "is-number": "^7.0.0" }, @@ -9644,8 +9590,7 @@ "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", - "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==" }, "node_modules/ts-jest": { "version": "29.1.1", @@ -10113,7 +10058,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -10219,7 +10163,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -10237,7 +10180,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -10253,14 +10195,12 @@ "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/wrap-ansi-cjs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -10274,7 +10214,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true, "engines": { "node": ">=12" }, @@ -10286,7 +10225,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, "engines": { "node": ">=12" }, @@ -10298,7 +10236,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -10397,7 +10334,6 @@ "version": "2.3.4", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==", - "dev": true, "engines": { "node": ">= 14" } diff --git a/app/src/app/api/download/[slug]/route.ts b/app/src/app/api/download/[slug]/route.ts index 96a5e77..3771043 100644 --- a/app/src/app/api/download/[slug]/route.ts +++ b/app/src/app/api/download/[slug]/route.ts @@ -1,14 +1,6 @@ -import * as env from "env-var"; -import * as Minio from "minio"; import { NextRequest } from "next/server"; -const minioClient = new Minio.Client({ - endPoint: env.get("MINIO_HOST").required().asString(), - port: 9000, - useSSL: false, - accessKey: env.get("MINIO_ACCESS_KEY").required().asString(), - secretKey: env.get("MINIO_SECRET_KEY").required().asString(), -}); +import minioClient from "../../../../lib/minioClient"; export async function GET( req: NextRequest, diff --git a/app/src/app/api/upload/route.ts b/app/src/app/api/upload/route.ts index b21662a..2fdaac0 100644 --- a/app/src/app/api/upload/route.ts +++ b/app/src/app/api/upload/route.ts @@ -1,17 +1,9 @@ import { getTemporalWorkflowClient } from "@/lib/temporalClient"; import { randomUUID } from "crypto"; -import * as env from "env-var"; -import * as Minio from "minio"; import { NextRequest } from "next/server"; import { extname } from "path"; -const minioClient = new Minio.Client({ - endPoint: env.get("MINIO_HOST").required().asString(), - port: 9000, - useSSL: false, - accessKey: env.get("MINIO_ACCESS_KEY").required().asString(), - secretKey: env.get("MINIO_SECRET_KEY").required().asString(), -}); +import minioClient from "../../../lib/minioClient"; export async function POST(req: NextRequest) { const formData = await req.formData(); From 4a44decd9fe22aaf8581fe3da718e4d5cbb5ca92 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 25 Jan 2024 10:46:43 +0100 Subject: [PATCH 4/4] chore(app): remove console.log --- app/temporal/src/activities.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts index ea20505..536b20b 100644 --- a/app/temporal/src/activities.ts +++ b/app/temporal/src/activities.ts @@ -229,7 +229,6 @@ export function makeActivities( ); const host = process.env.API_URL ?? "http://localhost:3000"; const downloadUrl = `${host}/api/download/${params.bucket}`; - console.log("downloadUrl", downloadUrl); fetch(params.callbackUrl, { method: "POST", body: downloadUrl,