From 1204136deca27f7363287151a589d988f7e6cf41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Tue, 23 Jan 2024 11:17:42 +0100
Subject: [PATCH 01/13] chore: handle cancellations

---
 app/temporal/src/workflows/importer.workflow.ts | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index d455be9..ec7252f 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -1,9 +1,11 @@
 import {
   ApplicationFailure,
+  CancellationScope,
   condition,
   defineQuery,
   defineSignal,
   defineUpdate,
+  isCancellation,
   proxyActivities,
   setHandler,
 } from "@temporalio/workflow";
@@ -172,10 +174,17 @@ export async function importer(params: ImporterWorkflowParams) {
       );
     }
   } catch (err) {
-    for (const compensation of compensations) {
-      await compensation();
+    const doCompensations = async () => {
+      for (const compensation of compensations) {
+        await compensation();
+      }
+    };
+    if (isCancellation(err)) {
+      await CancellationScope.nonCancellable(() => doCompensations());
+    } else {
+      await doCompensations();
     }
-    throw err;
+    throw err; // <-- Fail the workflow
   } finally {
     await acts.deleteBucket({ bucket: sourceFile!.bucket });
   }

From 1d38e21ba5a73682b85ab3a7fa2fca4e788a50b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Tue, 23 Jan 2024 17:09:15 +0100
Subject: [PATCH 02/13] feat: add hooks and empty response apis

---
 .../app/api/importer/[slug]/ImporterDto.ts    | 31 +++++++++++++
 .../app/api/importer/[slug]/patches/route.ts  | 14 ++++++
 .../api/importer/[slug]/source-data/route.ts  | 22 +++++++++
 .../api/importer/[slug]/validations/route.ts  | 22 +++++++++
 .../app/importer/[id]/validate/Validation.tsx | 38 ++++++++++++++-
 .../[id]/validate/ValidationTable.tsx         |  2 +
 app/src/app/importer/[id]/validate/page.tsx   | 46 +++++++++++++++++--
 app/src/components/hooks/useGetPatches.tsx    | 22 +++++++++
 app/src/components/hooks/useGetSourceData.tsx | 20 ++++++++
 .../components/hooks/useGetValidations.tsx    | 22 +++++++++
 app/src/lib/minioClient.ts                    | 12 +++++
 app/temporal/src/domain/DataSet.ts            |  7 ++-
 .../src/workflows/importer.workflow.ts        |  8 ++++
 13 files changed, 258 insertions(+), 8 deletions(-)
 create mode 100644 app/src/app/api/importer/[slug]/patches/route.ts
 create mode 100644 app/src/app/api/importer/[slug]/source-data/route.ts
 create mode 100644 app/src/app/api/importer/[slug]/validations/route.ts
 create mode 100644 app/src/components/hooks/useGetPatches.tsx
 create mode 100644 app/src/components/hooks/useGetSourceData.tsx
 create mode 100644 app/src/components/hooks/useGetValidations.tsx
 create mode 100644 app/src/lib/minioClient.ts

diff --git a/app/src/app/api/importer/[slug]/ImporterDto.ts b/app/src/app/api/importer/[slug]/ImporterDto.ts
index 4d974e4..832f829 100644
--- a/app/src/app/api/importer/[slug]/ImporterDto.ts
+++ b/app/src/app/api/importer/[slug]/ImporterDto.ts
@@ -29,6 +29,13 @@ export interface ImporterStatus {
   isImporting: boolean;
   dataMappingRecommendations: DataMappingRecommendation[] | null;
   dataMapping: DataMapping[] | null;
+
+  sourceData: {
+    bucket: string;
+    file: string;
+  } | null;
+
+  validations: { bucket: string; file: string } | null;
 }
 
 export interface DataMappingRecommendation {
@@ -51,3 +58,27 @@ export interface DataMapping {
   targetColumn: string | null;
   sourceColumn: string;
 }
+
+export interface DataSetPatch {
+  rowId: number;
+  /**
+   * target column
+   */
+  column: string;
+  newValue: string | number | null;
+}
+
+export interface DataValidation {
+  rowId: number;
+  column: string;
+  errors: ValidationError[];
+}
+
+export interface ValidationError {
+  type: "required" | "unique";
+  message: string;
+}
+
+export type SourceData = {
+  rowId: number;
+} & Record<string, string | number | null>;
diff --git a/app/src/app/api/importer/[slug]/patches/route.ts b/app/src/app/api/importer/[slug]/patches/route.ts
new file mode 100644
index 0000000..26194c1
--- /dev/null
+++ b/app/src/app/api/importer/[slug]/patches/route.ts
@@ -0,0 +1,14 @@
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { DataSetPatch } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } }
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<DataSetPatch[]>("importer:patches");
+  return NextResponse.json(result);
+}
diff --git a/app/src/app/api/importer/[slug]/source-data/route.ts b/app/src/app/api/importer/[slug]/source-data/route.ts
new file mode 100644
index 0000000..7012ba0
--- /dev/null
+++ b/app/src/app/api/importer/[slug]/source-data/route.ts
@@ -0,0 +1,22 @@
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { ImporterStatus } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } }
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<ImporterStatus>("importer:status");
+  //   if (!result.sourceData) {
+  //     return new NextResponse(undefined, { status: 404 });
+  //   }
+  //   const stream = await minioClient.getObject(
+  //     result.sourceData.bucket,
+  //     result.sourceData.file
+  //   );
+  //   return stream;
+  return NextResponse.json([]);
+}
diff --git a/app/src/app/api/importer/[slug]/validations/route.ts b/app/src/app/api/importer/[slug]/validations/route.ts
new file mode 100644
index 0000000..0c6ea75
--- /dev/null
+++ b/app/src/app/api/importer/[slug]/validations/route.ts
@@ -0,0 +1,22 @@
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { ImporterStatus } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } }
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<ImporterStatus>("importer:status");
+  //   if (!result.validations) {
+  //     return new NextResponse(undefined, { status: 404 });
+  //   }
+  //   const stream = await minioClient.getObject(
+  //     result.validations.bucket,
+  //     result.validations.file
+  //   );
+  //   return stream;
+  return NextResponse.json([]);
+}
diff --git a/app/src/app/importer/[id]/validate/Validation.tsx b/app/src/app/importer/[id]/validate/Validation.tsx
index 7ea6e3d..efa860a 100644
--- a/app/src/app/importer/[id]/validate/Validation.tsx
+++ b/app/src/app/importer/[id]/validate/Validation.tsx
@@ -1,13 +1,47 @@
-import { ImporterDto } from "@/app/api/importer/[slug]/ImporterDto";
+"use client";
+import {
+  DataSetPatch,
+  DataValidation,
+  ImporterDto,
+  SourceData,
+} from "@/app/api/importer/[slug]/ImporterDto";
+import { useGetImporter } from "@/components/hooks/useGetImporter";
+import { useGetPatches } from "@/components/hooks/useGetPatches";
+import { useGetSourceData } from "@/components/hooks/useGetSourceData";
+import { useGetValidations } from "@/components/hooks/useGetValidations";
 import { Button } from "@/components/ui/button";
 import { ChevronRightCircleIcon } from "lucide-react";
 import ValidationTable from "./ValidationTable";
 
 type Props = {
   initialImporterDto: ImporterDto;
+  initialSourceData: SourceData[];
+  initialPatches: DataSetPatch[];
+  initialValidation: DataValidation[];
 };
 
-const Validation = ({ initialImporterDto }: Props) => {
+const Validation = ({
+  initialImporterDto,
+  initialSourceData,
+  initialPatches,
+  initialValidation,
+}: Props) => {
+  const { importer } = useGetImporter(
+    initialImporterDto.importerId,
+    undefined,
+    initialImporterDto
+  );
+  const { data: sourceData } = useGetSourceData(initialImporterDto.importerId);
+  const { patches } = useGetPatches(
+    initialImporterDto.importerId,
+    undefined,
+    initialPatches
+  );
+  const { validations } = useGetValidations(
+    initialImporterDto.importerId,
+    undefined,
+    initialValidation
+  );
   return (
     <div>
       <h1 className="text-3xl font-bold mb-4">Validate your data</h1>
diff --git a/app/src/app/importer/[id]/validate/ValidationTable.tsx b/app/src/app/importer/[id]/validate/ValidationTable.tsx
index 2c406db..ade5ab4 100644
--- a/app/src/app/importer/[id]/validate/ValidationTable.tsx
+++ b/app/src/app/importer/[id]/validate/ValidationTable.tsx
@@ -19,6 +19,8 @@ import { useVirtualizer } from "@tanstack/react-virtual";
 import React from "react";
 import { makeData } from "./makeData";
 
+// TODO get patches and source data
+
 type Props = {
   importerDto: ImporterDto;
 };
diff --git a/app/src/app/importer/[id]/validate/page.tsx b/app/src/app/importer/[id]/validate/page.tsx
index 4c3692e..e71df1b 100644
--- a/app/src/app/importer/[id]/validate/page.tsx
+++ b/app/src/app/importer/[id]/validate/page.tsx
@@ -1,18 +1,56 @@
-import { ImporterDto } from "@/app/api/importer/[slug]/ImporterDto";
+import {
+  DataSetPatch,
+  DataValidation,
+  ImporterDto,
+  SourceData,
+} from "@/app/api/importer/[slug]/ImporterDto";
 import { getHost } from "@/lib/utils";
 import Validation from "./Validation";
 
 export default async function page(props: { params: { id: string } }) {
   const importerId = props.params.id;
-  const initialImporterDto = (await fetch(
+  const initialImporterDtoPromise = fetch(
     `${getHost()}/api/importer/${importerId}`,
     {
       cache: "no-cache",
     }
-  ).then((res) => res.json())) as ImporterDto;
+  ).then((res) => res.json() as Promise<ImporterDto>);
+  const initialPatchesPromise = fetch(
+    `${getHost()}/api/importer/${importerId}/patches`,
+    {
+      cache: "no-cache",
+    }
+  ).then((res) => res.json() as Promise<DataSetPatch[]>);
+  const initialValidationPromise = fetch(
+    `${getHost()}/api/importer/${importerId}/validations`,
+    {
+      cache: "no-cache",
+    }
+  ).then((res) => res.json() as Promise<DataValidation[]>);
+  const initialSourceDataPromise = fetch(
+    //! cached
+    `${getHost()}/api/importer/${importerId}/source-data`
+  ).then((res) => res.json() as Promise<SourceData[]>);
+  const [
+    initialImporterDto,
+    initialPatches,
+    initialValidation,
+    initialSourceData,
+  ] = await Promise.all([
+    initialImporterDtoPromise,
+    initialPatchesPromise,
+    initialValidationPromise,
+    initialSourceDataPromise,
+  ]);
+
   return (
     <div className="p-4">
-      <Validation initialImporterDto={initialImporterDto} />
+      <Validation
+        initialImporterDto={initialImporterDto}
+        initialPatches={initialPatches}
+        initialValidation={initialValidation}
+        initialSourceData={initialSourceData}
+      />
     </div>
   );
 }
diff --git a/app/src/components/hooks/useGetPatches.tsx b/app/src/components/hooks/useGetPatches.tsx
new file mode 100644
index 0000000..8c333bc
--- /dev/null
+++ b/app/src/components/hooks/useGetPatches.tsx
@@ -0,0 +1,22 @@
+import { DataSetPatch } from "@/app/api/importer/[slug]/ImporterDto";
+import useSWR from "swr";
+
+export function useGetPatches(
+  importerId: string | null,
+  pollInterval?: number,
+  fallbackData?: DataSetPatch[]
+) {
+  const { data, error, isLoading } = useSWR(
+    importerId ? [`/api/importer/${importerId}/patches`] : null,
+    ([url]) => fetch(url).then((res) => res.json()),
+    {
+      refreshInterval: pollInterval,
+      fallbackData,
+    }
+  );
+  return {
+    patches: data as DataSetPatch[],
+    error,
+    isLoading,
+  };
+}
diff --git a/app/src/components/hooks/useGetSourceData.tsx b/app/src/components/hooks/useGetSourceData.tsx
new file mode 100644
index 0000000..14291ef
--- /dev/null
+++ b/app/src/components/hooks/useGetSourceData.tsx
@@ -0,0 +1,20 @@
+import { SourceData } from "@/app/api/importer/[slug]/ImporterDto";
+import useSWR from "swr";
+
+export function useGetSourceData(
+  importerId: string | null,
+  fallbackData?: SourceData[]
+) {
+  const { data, error, isLoading } = useSWR(
+    importerId ? [`/api/importer/${importerId}/validations`] : null,
+    ([url]) => fetch(url).then((res) => res.json()),
+    {
+      fallbackData,
+    }
+  );
+  return {
+    data: data as SourceData[],
+    error,
+    isLoading,
+  };
+}
diff --git a/app/src/components/hooks/useGetValidations.tsx b/app/src/components/hooks/useGetValidations.tsx
new file mode 100644
index 0000000..218db4f
--- /dev/null
+++ b/app/src/components/hooks/useGetValidations.tsx
@@ -0,0 +1,22 @@
+import { DataValidation } from "@/app/api/importer/[slug]/ImporterDto";
+import useSWR from "swr";
+
+export function useGetValidations(
+  importerId: string | null,
+  pollInterval?: number,
+  fallbackData?: DataValidation[]
+) {
+  const { data, error, isLoading } = useSWR(
+    importerId ? [`/api/importer/${importerId}/validations`] : null,
+    ([url]) => fetch(url).then((res) => res.json()),
+    {
+      refreshInterval: pollInterval,
+      fallbackData,
+    }
+  );
+  return {
+    validations: data as DataValidation[],
+    error,
+    isLoading,
+  };
+}
diff --git a/app/src/lib/minioClient.ts b/app/src/lib/minioClient.ts
new file mode 100644
index 0000000..99aae3a
--- /dev/null
+++ b/app/src/lib/minioClient.ts
@@ -0,0 +1,12 @@
+import env from "env-var";
+import * as Minio from "minio";
+
+const minioClient = new Minio.Client({
+  endPoint: env.get("MINIO_HOST").required().asString(),
+  port: 9000,
+  useSSL: false,
+  accessKey: env.get("MINIO_ACCESS_KEY").required().asString(),
+  secretKey: env.get("MINIO_SECRET_KEY").required().asString(),
+});
+
+export default minioClient;
diff --git a/app/temporal/src/domain/DataSet.ts b/app/temporal/src/domain/DataSet.ts
index 1263d07..c72c666 100644
--- a/app/temporal/src/domain/DataSet.ts
+++ b/app/temporal/src/domain/DataSet.ts
@@ -1,5 +1,8 @@
 export interface DataSetPatch {
-  row: number;
-  col: number;
+  rowId: number;
+  /**
+   * target column
+   */
+  column: string;
   newValue: string | number | null;
 }
diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index ec7252f..c49e7a5 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -37,6 +37,8 @@ export interface ImporterStatus {
   isWaitingForImport: boolean;
   isImporting: boolean;
   dataMappingRecommendations: DataMappingRecommendation[] | null;
+  sourceData: { bucket: string; file: string } | null;
+  validations: { bucket: string; file: string } | null;
 }
 
 export interface Mapping {
@@ -68,6 +70,7 @@ const mappingUpdate = defineUpdate<
     }
   ]
 >("importer:update-mapping");
+const importPatchesQuery = defineQuery<DataSetPatch[]>("importer:patches");
 
 const acts = proxyActivities<ReturnType<typeof makeActivities>>({
   startToCloseTimeout: "5 minute",
@@ -134,8 +137,13 @@ export async function importer(params: ImporterWorkflowParams) {
       isImporting: importStartRequested === true,
       dataMappingRecommendations,
       dataMapping: configuredMappings,
+      sourceData: null, // TODO return merged and mapped source data {bucket,file}
+      validations: null, // TODO return current validations {bucket,file}
     };
   });
+  setHandler(importPatchesQuery, () => {
+    return patches;
+  });
 
   try {
     const hasSourceFile = await condition(

From 43e470760b75d758903e784efc4edecfed385800 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 08:07:38 +0100
Subject: [PATCH 03/13] fix: update mapping api handler

---
 app/src/app/api/importer/[slug]/mappings/route.ts | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/src/app/api/importer/[slug]/mappings/route.ts b/app/src/app/api/importer/[slug]/mappings/route.ts
index 6bfc3db..896ce36 100644
--- a/app/src/app/api/importer/[slug]/mappings/route.ts
+++ b/app/src/app/api/importer/[slug]/mappings/route.ts
@@ -1,15 +1,14 @@
 import { getTemporalWorkflowClient } from "@/lib/temporalClient";
 import { NextRequest, NextResponse } from "next/server";
-import { DataMapping } from "../ImporterDto";
 
 export async function PUT(
-  _req: NextRequest,
-  { params, body }: { params: { slug: string }; body: DataMapping[] }
+  req: NextRequest,
+  { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
   const client = getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
-  const mappings = body;
+  const mappings = await req.json();
   await handle.executeUpdate("importer:update-mapping", {
     args: [{ mappings }],
   });

From ff9bf8ca6ade755b3707e73ba33ba3b51a196ad3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 09:38:35 +0100
Subject: [PATCH 04/13] chore: re-add await for mappings

---
 app/temporal/src/workflows/importer.workflow.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index 1836fac..62c3d3c 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -177,7 +177,7 @@ export async function importer(params: ImporterWorkflowParams) {
       columnConfig: params.columnConfig,
     });
 
-    // await condition(() => configuredMappings !== null, startImportTimeout);
+    await condition(() => configuredMappings !== null, startImportTimeout);
 
     const chunkedFileReferences = await acts.applyMappings({
       bucket: sourceFile!.bucket,

From c7b325ffb5c776bcf8a9691dc5b6062e79a4aa69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 14:10:26 +0100
Subject: [PATCH 05/13] feat: get data from backend

---
 app/package-lock.json                         | 400 ++++++++++--------
 .../app/api/importer/[slug]/ImporterDto.ts    |   4 +-
 .../api/importer/[slug]/source-data/route.ts  |  33 +-
 .../api/importer/[slug]/validations/route.ts  |  30 +-
 .../importer/[id]/mapping/ShowMappings.tsx    |  15 +-
 .../app/importer/[id]/validate/Validation.tsx |  13 +-
 .../[id]/validate/ValidationTable.tsx         |  18 +-
 app/src/app/importer/[id]/validate/page.tsx   |   6 +-
 app/src/components/hooks/useGetSourceData.tsx |   2 +-
 .../src/workflows/importer.workflow.ts        |  49 ++-
 10 files changed, 356 insertions(+), 214 deletions(-)

diff --git a/app/package-lock.json b/app/package-lock.json
index 7dd57de..52ec471 100644
--- a/app/package-lock.json
+++ b/app/package-lock.json
@@ -66,6 +66,7 @@
     },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",
+      "dev": true,
       "license": "MIT",
       "engines": {
         "node": ">=10"
@@ -875,6 +876,7 @@
     },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
+      "dev": true,
       "license": "ISC",
       "dependencies": {
         "string-width": "^5.1.2",
@@ -890,6 +892,7 @@
     },
     "node_modules/@isaacs/cliui/node_modules/ansi-regex": {
       "version": "6.0.1",
+      "dev": true,
       "license": "MIT",
       "engines": {
         "node": ">=12"
@@ -900,6 +903,7 @@
     },
     "node_modules/@isaacs/cliui/node_modules/strip-ansi": {
       "version": "7.1.0",
+      "dev": true,
       "license": "MIT",
       "dependencies": {
         "ansi-regex": "^6.0.1"
@@ -1393,10 +1397,131 @@
         "node": ">= 10"
       }
     },
+    "node_modules/@next/swc-darwin-x64": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz",
+      "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-linux-arm64-gnu": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz",
+      "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-linux-arm64-musl": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz",
+      "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-linux-x64-gnu": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz",
+      "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-linux-x64-musl": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz",
+      "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-win32-arm64-msvc": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz",
+      "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-win32-ia32-msvc": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz",
+      "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==",
+      "cpu": [
+        "ia32"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/@next/swc-win32-x64-msvc": {
+      "version": "14.1.0",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz",
+      "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
       "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
       "dependencies": {
         "@nodelib/fs.stat": "2.0.5",
         "run-parallel": "^1.1.9"
@@ -1409,6 +1534,7 @@
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
       "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
       "engines": {
         "node": ">= 8"
       }
@@ -1417,6 +1543,7 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
       "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
       "dependencies": {
         "@nodelib/fs.scandir": "2.1.5",
         "fastq": "^1.6.0"
@@ -1429,6 +1556,7 @@
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
       "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "dev": true,
       "optional": true,
       "engines": {
         "node": ">=14"
@@ -2401,13 +2529,13 @@
       "version": "15.7.11",
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
       "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==",
-      "devOptional": true
+      "dev": true
     },
     "node_modules/@types/react": {
       "version": "18.2.48",
       "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz",
       "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==",
-      "devOptional": true,
+      "dev": true,
       "dependencies": {
         "@types/prop-types": "*",
         "@types/scheduler": "*",
@@ -2418,7 +2546,7 @@
       "version": "18.2.18",
       "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.18.tgz",
       "integrity": "sha512-TJxDm6OfAX2KJWJdMEVTwWke5Sc/E/RlnPGvGfS0W7+6ocy2xhDVQVh/KvC2Uf7kACs+gDytdusDSdWfWkaNzw==",
-      "devOptional": true,
+      "dev": true,
       "dependencies": {
         "@types/react": "*"
       }
@@ -2427,7 +2555,7 @@
       "version": "0.16.8",
       "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
       "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==",
-      "devOptional": true
+      "dev": true
     },
     "node_modules/@types/stack-utils": {
       "version": "2.0.3",
@@ -2858,12 +2986,14 @@
     "node_modules/any-promise": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
-      "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A=="
+      "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==",
+      "dev": true
     },
     "node_modules/anymatch": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
       "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
+      "dev": true,
       "dependencies": {
         "normalize-path": "^3.0.0",
         "picomatch": "^2.0.4"
@@ -3238,12 +3368,14 @@
     "node_modules/balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "dev": true
     },
     "node_modules/binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
       "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
+      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -3270,6 +3402,7 @@
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
       "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "dev": true,
       "dependencies": {
         "fill-range": "^7.0.1"
       },
@@ -3393,6 +3526,7 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz",
       "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==",
+      "dev": true,
       "engines": {
         "node": ">= 6"
       }
@@ -3465,6 +3599,7 @@
       "version": "3.5.3",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
       "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
+      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -3491,6 +3626,7 @@
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
       "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "dev": true,
       "dependencies": {
         "is-glob": "^4.0.1"
       },
@@ -3650,6 +3786,7 @@
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
       "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
+      "dev": true,
       "engines": {
         "node": ">= 6"
       }
@@ -3702,6 +3839,7 @@
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
       "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dev": true,
       "dependencies": {
         "path-key": "^3.1.0",
         "shebang-command": "^2.0.0",
@@ -3715,6 +3853,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
       "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==",
+      "dev": true,
       "bin": {
         "cssesc": "bin/cssesc"
       },
@@ -3726,7 +3865,7 @@
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
       "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
-      "devOptional": true
+      "dev": true
     },
     "node_modules/csv": {
       "version": "6.3.6",
@@ -3873,7 +4012,8 @@
     "node_modules/didyoumean": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz",
-      "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw=="
+      "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==",
+      "dev": true
     },
     "node_modules/diff-sequences": {
       "version": "29.6.3",
@@ -3899,7 +4039,8 @@
     "node_modules/dlv": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
-      "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA=="
+      "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==",
+      "dev": true
     },
     "node_modules/doctrine": {
       "version": "3.0.0",
@@ -3927,7 +4068,8 @@
     "node_modules/eastasianwidth": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
+      "dev": true
     },
     "node_modules/electron-to-chromium": {
       "version": "1.4.639",
@@ -3949,7 +4091,8 @@
     "node_modules/emoji-regex": {
       "version": "9.2.2",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
+      "dev": true
     },
     "node_modules/enhanced-resolve": {
       "version": "5.15.0",
@@ -4616,15 +4759,11 @@
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
       "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
     },
-    "node_modules/fast-diff": {
-      "version": "1.3.0",
-      "license": "Apache-2.0",
-      "peer": true
-    },
     "node_modules/fast-glob": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
       "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
+      "dev": true,
       "dependencies": {
         "@nodelib/fs.stat": "^2.0.2",
         "@nodelib/fs.walk": "^1.2.3",
@@ -4640,6 +4779,7 @@
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
       "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "dev": true,
       "dependencies": {
         "is-glob": "^4.0.1"
       },
@@ -4683,6 +4823,7 @@
       "version": "1.16.0",
       "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.16.0.tgz",
       "integrity": "sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==",
+      "dev": true,
       "dependencies": {
         "reusify": "^1.0.4"
       }
@@ -4712,6 +4853,7 @@
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
       "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
       },
@@ -4775,6 +4917,7 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
       "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
+      "dev": true,
       "dependencies": {
         "cross-spawn": "^7.0.0",
         "signal-exit": "^4.0.1"
@@ -4822,6 +4965,7 @@
       "version": "2.3.3",
       "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
       "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
+      "dev": true,
       "hasInstallScript": true,
       "optional": true,
       "os": [
@@ -4966,6 +5110,7 @@
       "version": "10.3.10",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
       "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
+      "dev": true,
       "dependencies": {
         "foreground-child": "^3.1.0",
         "jackspeak": "^2.3.5",
@@ -4987,6 +5132,7 @@
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
       "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
       "dependencies": {
         "is-glob": "^4.0.3"
       },
@@ -5003,6 +5149,7 @@
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
       "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dev": true,
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -5011,6 +5158,7 @@
       "version": "9.0.3",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz",
       "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==",
+      "dev": true,
       "dependencies": {
         "brace-expansion": "^2.0.1"
       },
@@ -5395,6 +5543,7 @@
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
       "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+      "dev": true,
       "dependencies": {
         "binary-extensions": "^2.0.0"
       },
@@ -5433,6 +5582,7 @@
       "version": "2.13.1",
       "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz",
       "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==",
+      "dev": true,
       "dependencies": {
         "hasown": "^2.0.0"
       },
@@ -5459,6 +5609,7 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
       "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -5510,6 +5661,7 @@
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
       "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+      "dev": true,
       "dependencies": {
         "is-extglob": "^2.1.1"
       },
@@ -5542,6 +5694,7 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
       "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+      "dev": true,
       "engines": {
         "node": ">=0.12.0"
       }
@@ -5706,7 +5859,8 @@
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+      "dev": true
     },
     "node_modules/istanbul-lib-coverage": {
       "version": "3.2.2",
@@ -5800,6 +5954,7 @@
       "version": "2.3.6",
       "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
       "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
+      "dev": true,
       "dependencies": {
         "@isaacs/cliui": "^8.0.2"
       },
@@ -6515,6 +6670,7 @@
       "version": "1.21.0",
       "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz",
       "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==",
+      "dev": true,
       "bin": {
         "jiti": "bin/jiti.js"
       }
@@ -6704,6 +6860,7 @@
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz",
       "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==",
+      "dev": true,
       "engines": {
         "node": ">=10"
       }
@@ -6711,7 +6868,8 @@
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
-      "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
+      "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
+      "dev": true
     },
     "node_modules/load-json-file": {
       "version": "4.0.0",
@@ -6770,16 +6928,6 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
-    "node_modules/lodash.clonedeep": {
-      "version": "4.5.0",
-      "license": "MIT",
-      "peer": true
-    },
-    "node_modules/lodash.isequal": {
-      "version": "4.5.0",
-      "license": "MIT",
-      "peer": true
-    },
     "node_modules/lodash.memoize": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@@ -6812,6 +6960,7 @@
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz",
       "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==",
+      "dev": true,
       "engines": {
         "node": "14 || >=16.14"
       }
@@ -6891,6 +7040,7 @@
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
       "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "dev": true,
       "engines": {
         "node": ">= 8"
       }
@@ -6899,6 +7049,7 @@
       "version": "4.0.5",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
       "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
+      "dev": true,
       "dependencies": {
         "braces": "^3.0.2",
         "picomatch": "^2.3.1"
@@ -6984,6 +7135,7 @@
       "version": "7.0.4",
       "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
       "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
+      "dev": true,
       "engines": {
         "node": ">=16 || 14 >=14.17"
       }
@@ -6998,6 +7150,7 @@
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
       "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==",
+      "dev": true,
       "dependencies": {
         "any-promise": "^1.0.0",
         "object-assign": "^4.0.1",
@@ -7210,6 +7363,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -7402,6 +7556,7 @@
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
       "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -7410,6 +7565,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
       "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
+      "dev": true,
       "engines": {
         "node": ">= 6"
       }
@@ -7649,6 +7805,7 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
       "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -7656,12 +7813,14 @@
     "node_modules/path-parse": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
-      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "dev": true
     },
     "node_modules/path-scurry": {
       "version": "1.10.1",
       "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
       "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==",
+      "dev": true,
       "dependencies": {
         "lru-cache": "^9.1.1 || ^10.0.0",
         "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
@@ -7691,6 +7850,7 @@
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
       "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "dev": true,
       "engines": {
         "node": ">=8.6"
       },
@@ -7714,6 +7874,7 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
       "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -7722,6 +7883,7 @@
       "version": "4.0.6",
       "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz",
       "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==",
+      "dev": true,
       "engines": {
         "node": ">= 6"
       }
@@ -7794,6 +7956,7 @@
       "version": "8.4.33",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz",
       "integrity": "sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==",
+      "dev": true,
       "funding": [
         {
           "type": "opencollective",
@@ -7821,6 +7984,7 @@
       "version": "15.1.0",
       "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz",
       "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==",
+      "dev": true,
       "dependencies": {
         "postcss-value-parser": "^4.0.0",
         "read-cache": "^1.0.0",
@@ -7837,6 +8001,7 @@
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz",
       "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==",
+      "dev": true,
       "dependencies": {
         "camelcase-css": "^2.0.1"
       },
@@ -7855,6 +8020,7 @@
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.2.tgz",
       "integrity": "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==",
+      "dev": true,
       "funding": [
         {
           "type": "opencollective",
@@ -7889,6 +8055,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.0.0.tgz",
       "integrity": "sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==",
+      "dev": true,
       "engines": {
         "node": ">=14"
       }
@@ -7897,6 +8064,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.1.tgz",
       "integrity": "sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==",
+      "dev": true,
       "dependencies": {
         "postcss-selector-parser": "^6.0.11"
       },
@@ -7915,6 +8083,7 @@
       "version": "6.0.15",
       "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.15.tgz",
       "integrity": "sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==",
+      "dev": true,
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"
@@ -7926,7 +8095,8 @@
     "node_modules/postcss-value-parser": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
-      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
+      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
+      "dev": true
     },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
@@ -8078,6 +8248,7 @@
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
       "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -8093,19 +8264,6 @@
         }
       ]
     },
-    "node_modules/quill-delta": {
-      "version": "5.1.0",
-      "license": "MIT",
-      "peer": true,
-      "dependencies": {
-        "fast-diff": "^1.3.0",
-        "lodash.clonedeep": "^4.5.0",
-        "lodash.isequal": "^4.5.0"
-      },
-      "engines": {
-        "node": ">= 12.0.0"
-      }
-    },
     "node_modules/randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -8214,6 +8372,7 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
       "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==",
+      "dev": true,
       "dependencies": {
         "pify": "^2.3.0"
       }
@@ -8270,6 +8429,7 @@
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
       "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
+      "dev": true,
       "dependencies": {
         "picomatch": "^2.2.1"
       },
@@ -8331,6 +8491,7 @@
       "version": "1.22.8",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
       "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
+      "dev": true,
       "dependencies": {
         "is-core-module": "^2.13.0",
         "path-parse": "^1.0.7",
@@ -8395,6 +8556,7 @@
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
       "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true,
       "engines": {
         "iojs": ">=1.0.0",
         "node": ">=0.10.0"
@@ -8439,6 +8601,7 @@
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
       "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -8622,6 +8785,7 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
       "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dev": true,
       "dependencies": {
         "shebang-regex": "^3.0.0"
       },
@@ -8633,6 +8797,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
       "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -8664,6 +8829,7 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
       "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+      "dev": true,
       "engines": {
         "node": ">=14"
       },
@@ -8874,6 +9040,7 @@
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
       "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dev": true,
       "dependencies": {
         "eastasianwidth": "^0.2.0",
         "emoji-regex": "^9.2.2",
@@ -8891,6 +9058,7 @@
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
       "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dev": true,
       "dependencies": {
         "emoji-regex": "^8.0.0",
         "is-fullwidth-code-point": "^3.0.0",
@@ -8903,12 +9071,14 @@
     "node_modules/string-width-cjs/node_modules/emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true
     },
     "node_modules/string-width/node_modules/ansi-regex": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
       "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
+      "dev": true,
       "engines": {
         "node": ">=12"
       },
@@ -8920,6 +9090,7 @@
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
       "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dev": true,
       "dependencies": {
         "ansi-regex": "^6.0.1"
       },
@@ -9028,6 +9199,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
       "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dev": true,
       "dependencies": {
         "ansi-regex": "^5.0.1"
       },
@@ -9096,6 +9268,7 @@
       "version": "3.35.0",
       "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz",
       "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==",
+      "dev": true,
       "dependencies": {
         "@jridgewell/gen-mapping": "^0.3.2",
         "commander": "^4.0.0",
@@ -9129,6 +9302,7 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
       "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "dev": true,
       "engines": {
         "node": ">= 0.4"
       },
@@ -9173,6 +9347,7 @@
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.1.tgz",
       "integrity": "sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==",
+      "dev": true,
       "dependencies": {
         "@alloc/quick-lru": "^5.2.0",
         "arg": "^5.0.2",
@@ -9320,6 +9495,7 @@
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
       "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==",
+      "dev": true,
       "dependencies": {
         "any-promise": "^1.0.0"
       }
@@ -9328,6 +9504,7 @@
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
       "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==",
+      "dev": true,
       "dependencies": {
         "thenify": ">= 3.1.0 < 4"
       },
@@ -9373,6 +9550,7 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
       "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+      "dev": true,
       "dependencies": {
         "is-number": "^7.0.0"
       },
@@ -9407,7 +9585,8 @@
     "node_modules/ts-interface-checker": {
       "version": "0.1.13",
       "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz",
-      "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA=="
+      "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==",
+      "dev": true
     },
     "node_modules/ts-jest": {
       "version": "29.1.1",
@@ -9875,6 +10054,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
       "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dev": true,
       "dependencies": {
         "isexe": "^2.0.0"
       },
@@ -9980,6 +10160,7 @@
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
       "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dev": true,
       "dependencies": {
         "ansi-styles": "^6.1.0",
         "string-width": "^5.0.1",
@@ -9997,6 +10178,7 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
       "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dev": true,
       "dependencies": {
         "ansi-styles": "^4.0.0",
         "string-width": "^4.1.0",
@@ -10012,12 +10194,14 @@
     "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "dev": true
     },
     "node_modules/wrap-ansi-cjs/node_modules/string-width": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
       "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dev": true,
       "dependencies": {
         "emoji-regex": "^8.0.0",
         "is-fullwidth-code-point": "^3.0.0",
@@ -10031,6 +10215,7 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
       "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
+      "dev": true,
       "engines": {
         "node": ">=12"
       },
@@ -10042,6 +10227,7 @@
       "version": "6.2.1",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
       "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "dev": true,
       "engines": {
         "node": ">=12"
       },
@@ -10053,6 +10239,7 @@
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
       "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
+      "dev": true,
       "dependencies": {
         "ansi-regex": "^6.0.1"
       },
@@ -10151,6 +10338,7 @@
       "version": "2.3.4",
       "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz",
       "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==",
+      "dev": true,
       "engines": {
         "node": ">= 14"
       }
@@ -10208,126 +10396,6 @@
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
       }
-    },
-    "node_modules/@next/swc-darwin-x64": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz",
-      "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz",
-      "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz",
-      "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz",
-      "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz",
-      "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz",
-      "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz",
-      "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==",
-      "cpu": [
-        "ia32"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
-    "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz",
-      "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
     }
   }
 }
diff --git a/app/src/app/api/importer/[slug]/ImporterDto.ts b/app/src/app/api/importer/[slug]/ImporterDto.ts
index 832f829..f7cbc60 100644
--- a/app/src/app/api/importer/[slug]/ImporterDto.ts
+++ b/app/src/app/api/importer/[slug]/ImporterDto.ts
@@ -32,10 +32,10 @@ export interface ImporterStatus {
 
   sourceData: {
     bucket: string;
-    file: string;
+    fileReference: string;
   } | null;
 
-  validations: { bucket: string; file: string } | null;
+  validations: { bucket: string; fileReference: string } | null;
 }
 
 export interface DataMappingRecommendation {
diff --git a/app/src/app/api/importer/[slug]/source-data/route.ts b/app/src/app/api/importer/[slug]/source-data/route.ts
index 7012ba0..72767bd 100644
--- a/app/src/app/api/importer/[slug]/source-data/route.ts
+++ b/app/src/app/api/importer/[slug]/source-data/route.ts
@@ -1,22 +1,35 @@
+import minioClient from "@/lib/minioClient";
 import { getTemporalWorkflowClient } from "@/lib/temporalClient";
 import { NextRequest, NextResponse } from "next/server";
 import { ImporterStatus } from "../ImporterDto";
 
 export async function GET(
   _req: NextRequest,
-  { params }: { params: { slug: string } }
+  { params }: { params: { slug: string } },
+  res: NextResponse
 ) {
   const { slug: importerId } = params;
   const client = getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const result = await handle.query<ImporterStatus>("importer:status");
-  //   if (!result.sourceData) {
-  //     return new NextResponse(undefined, { status: 404 });
-  //   }
-  //   const stream = await minioClient.getObject(
-  //     result.sourceData.bucket,
-  //     result.sourceData.file
-  //   );
-  //   return stream;
-  return NextResponse.json([]);
+  if (!result.sourceData) {
+    return new NextResponse(undefined, { status: 404 });
+  }
+  const stream = await minioClient.getObject(
+    result.sourceData.bucket,
+    result.sourceData.fileReference
+  );
+  // TODO find better to directly return stream...
+  const buffers = [];
+  // node.js readable streams implement the async iterator protocol
+  for await (const data of stream) {
+    buffers.push(data);
+  }
+  const fileAsBuffer = Buffer.concat(buffers);
+  return new Response(fileAsBuffer, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
 }
diff --git a/app/src/app/api/importer/[slug]/validations/route.ts b/app/src/app/api/importer/[slug]/validations/route.ts
index 0c6ea75..87f9f2a 100644
--- a/app/src/app/api/importer/[slug]/validations/route.ts
+++ b/app/src/app/api/importer/[slug]/validations/route.ts
@@ -1,3 +1,4 @@
+import minioClient from "@/lib/minioClient";
 import { getTemporalWorkflowClient } from "@/lib/temporalClient";
 import { NextRequest, NextResponse } from "next/server";
 import { ImporterStatus } from "../ImporterDto";
@@ -10,13 +11,24 @@ export async function GET(
   const client = getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const result = await handle.query<ImporterStatus>("importer:status");
-  //   if (!result.validations) {
-  //     return new NextResponse(undefined, { status: 404 });
-  //   }
-  //   const stream = await minioClient.getObject(
-  //     result.validations.bucket,
-  //     result.validations.file
-  //   );
-  //   return stream;
-  return NextResponse.json([]);
+  if (!result.validations) {
+    return new NextResponse("not found", { status: 404 });
+  }
+  const stream = await minioClient.getObject(
+    result.validations.bucket,
+    result.validations.fileReference
+  );
+  // TODO find better to directly return stream...
+  const buffers = [];
+  // node.js readable streams implement the async iterator protocol
+  for await (const data of stream) {
+    buffers.push(data);
+  }
+  const fileAsBuffer = Buffer.concat(buffers);
+  return new Response(fileAsBuffer, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
 }
diff --git a/app/src/app/importer/[id]/mapping/ShowMappings.tsx b/app/src/app/importer/[id]/mapping/ShowMappings.tsx
index 87c215c..1cbeb59 100644
--- a/app/src/app/importer/[id]/mapping/ShowMappings.tsx
+++ b/app/src/app/importer/[id]/mapping/ShowMappings.tsx
@@ -34,14 +34,27 @@ interface Mapping {
 
 const ShowMappings = ({ importerDto: initialImporterDto }: Props) => {
   const { push } = useRouter();
+  const [enablePolling, setEnablePolling] = React.useState(false);
   const { importer } = useGetImporter(
     initialImporterDto.importerId,
-    undefined,
+    enablePolling ? 1000 : undefined,
     initialImporterDto
   );
   const {
     status: { dataMappingRecommendations },
   } = importer;
+
+  React.useEffect(() => {
+    if (
+      !dataMappingRecommendations ||
+      dataMappingRecommendations.length === 0
+    ) {
+      setEnablePolling(true);
+    } else {
+      setEnablePolling(false);
+    }
+  }, [dataMappingRecommendations]);
+
   const [currentMappings, setCurrentMappings] = React.useState<Mapping[]>([]);
   React.useEffect(() => {
     const newMapping =
diff --git a/app/src/app/importer/[id]/validate/Validation.tsx b/app/src/app/importer/[id]/validate/Validation.tsx
index efa860a..32632d6 100644
--- a/app/src/app/importer/[id]/validate/Validation.tsx
+++ b/app/src/app/importer/[id]/validate/Validation.tsx
@@ -31,7 +31,10 @@ const Validation = ({
     undefined,
     initialImporterDto
   );
-  const { data: sourceData } = useGetSourceData(initialImporterDto.importerId);
+  const { data: sourceData } = useGetSourceData(
+    initialImporterDto.importerId,
+    initialSourceData
+  );
   const { patches } = useGetPatches(
     initialImporterDto.importerId,
     undefined,
@@ -42,11 +45,17 @@ const Validation = ({
     undefined,
     initialValidation
   );
+  console.log(sourceData, patches, validations);
   return (
     <div>
       <h1 className="text-3xl font-bold mb-4">Validate your data</h1>
       <div className="mb-4">
-        <ValidationTable importerDto={initialImporterDto} />
+        <ValidationTable
+          importerDto={importer}
+          data={sourceData}
+          validations={validations}
+          patches={patches}
+        />
       </div>
       <div className="flex justify-end">
         <Button>
diff --git a/app/src/app/importer/[id]/validate/ValidationTable.tsx b/app/src/app/importer/[id]/validate/ValidationTable.tsx
index ade5ab4..f750972 100644
--- a/app/src/app/importer/[id]/validate/ValidationTable.tsx
+++ b/app/src/app/importer/[id]/validate/ValidationTable.tsx
@@ -1,5 +1,10 @@
 "use client";
-import { ImporterDto } from "@/app/api/importer/[slug]/ImporterDto";
+import {
+  DataSetPatch,
+  DataValidation,
+  ImporterDto,
+  SourceData,
+} from "@/app/api/importer/[slug]/ImporterDto";
 import {
   Table,
   TableBody,
@@ -17,12 +22,12 @@ import {
 } from "@tanstack/react-table";
 import { useVirtualizer } from "@tanstack/react-virtual";
 import React from "react";
-import { makeData } from "./makeData";
-
-// TODO get patches and source data
 
 type Props = {
   importerDto: ImporterDto;
+  data: SourceData[];
+  validations: DataValidation[];
+  patches: DataSetPatch[];
 };
 
 const ValidationTable = (props: Props) => {
@@ -39,7 +44,10 @@ const ValidationTable = (props: Props) => {
       ),
     [props.importerDto.config.columnConfig]
   );
-  const data: Record<string, string>[] = React.useMemo(() => makeData(), []);
+  const data = React.useMemo(() => {
+    // TODO merge with patches
+    return props.data;
+  }, [props.data]);
   const table = useReactTable({
     data,
     columns,
diff --git a/app/src/app/importer/[id]/validate/page.tsx b/app/src/app/importer/[id]/validate/page.tsx
index e71df1b..fe8fbce 100644
--- a/app/src/app/importer/[id]/validate/page.tsx
+++ b/app/src/app/importer/[id]/validate/page.tsx
@@ -26,7 +26,10 @@ export default async function page(props: { params: { id: string } }) {
     {
       cache: "no-cache",
     }
-  ).then((res) => res.json() as Promise<DataValidation[]>);
+  ).then((res) => {
+    if (res.status === 404) return Promise.resolve([]);
+    return res.json() as Promise<DataValidation[]>;
+  });
   const initialSourceDataPromise = fetch(
     //! cached
     `${getHost()}/api/importer/${importerId}/source-data`
@@ -42,7 +45,6 @@ export default async function page(props: { params: { id: string } }) {
     initialValidationPromise,
     initialSourceDataPromise,
   ]);
-
   return (
     <div className="p-4">
       <Validation
diff --git a/app/src/components/hooks/useGetSourceData.tsx b/app/src/components/hooks/useGetSourceData.tsx
index 14291ef..ceac446 100644
--- a/app/src/components/hooks/useGetSourceData.tsx
+++ b/app/src/components/hooks/useGetSourceData.tsx
@@ -6,7 +6,7 @@ export function useGetSourceData(
   fallbackData?: SourceData[]
 ) {
   const { data, error, isLoading } = useSWR(
-    importerId ? [`/api/importer/${importerId}/validations`] : null,
+    importerId ? [`/api/importer/${importerId}/source-data`] : null,
     ([url]) => fetch(url).then((res) => res.json()),
     {
       fallbackData,
diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index 62c3d3c..5960da7 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -38,8 +38,8 @@ export interface ImporterStatus {
   isWaitingForImport: boolean;
   isImporting: boolean;
   dataMappingRecommendations: DataMappingRecommendation[] | null;
-  sourceData: { bucket: string; file: string } | null;
-  validations: { bucket: string; file: string } | null;
+  sourceData: { bucket: string; fileReference: string } | null;
+  validations: { bucket: string; fileReference: string } | null;
 }
 
 export interface Mapping {
@@ -96,6 +96,14 @@ export async function importer(params: ImporterWorkflowParams) {
   let validationFileReferences: string[] | null = null;
   let isValidating = false;
   let configuredMappings: Mapping[] | null = null;
+  let mappedSourceData: {
+    bucket: string;
+    fileReference: string;
+  } | null = null;
+  let latestValidations: {
+    bucket: string;
+    fileReference: string;
+  } | null = null;
 
   setHandler(
     addFileUpdate,
@@ -115,10 +123,11 @@ export async function importer(params: ImporterWorkflowParams) {
   setHandler(
     mappingUpdate,
     (params) => {
+      console.log("handle update mapping", params);
       configuredMappings = params.mappings;
     },
     {
-      validator: (params) => {
+      validator: (_params) => {
         return !configuredMappings;
       },
     }
@@ -142,8 +151,8 @@ export async function importer(params: ImporterWorkflowParams) {
       isValidating,
       validationFileReferences,
       dataMapping: configuredMappings,
-      sourceData: null, // TODO return merged and mapped source data {bucket,file}
-      validations: null, // TODO return current validations {bucket,file}
+      sourceData: mappedSourceData,
+      validations: latestValidations,
     };
   });
   setHandler(importPatchesQuery, () => {
@@ -187,7 +196,16 @@ export async function importer(params: ImporterWorkflowParams) {
         targetColumn: item.targetColumn,
       })),
     });
-
+    const mappedSourceDataFileReference = "target.json";
+    await acts.mergeChunks({
+      bucket: sourceFile!.bucket,
+      fileReferences: chunkedFileReferences,
+      outputFileReference: mappedSourceDataFileReference,
+    });
+    mappedSourceData = {
+      bucket: sourceFile!.bucket,
+      fileReference: mappedSourceDataFileReference,
+    };
     await performValidations(sourceFileReference, chunkedFileReferences);
 
     const hasImportStartRequested = await condition(
@@ -207,7 +225,7 @@ export async function importer(params: ImporterWorkflowParams) {
     };
     if (isCancellation(err)) {
       await CancellationScope.nonCancellable(() => doCompensations());
-    } else {
+    } else if (err instanceof ApplicationFailure && err.nonRetryable) {
       await doCompensations();
     }
     throw err; // <-- Fail the workflow
@@ -249,7 +267,7 @@ export async function importer(params: ImporterWorkflowParams) {
       bucket: sourceFile!.bucket,
       fileReference: sourceFileReference,
       outputFileReference: statsFileReference,
-      uniqueColumns: validatorColumns.unique.map((item) => item.column),
+      uniqueColumns: validatorColumns.unique?.map((item) => item.column) ?? [],
     });
     console.log(`generate stats file took ${Date.now() - startStats}ms`);
 
@@ -265,18 +283,17 @@ export async function importer(params: ImporterWorkflowParams) {
         })
       )
     );
-    const errorFileReferences = await Promise.all(parallelValidations);
-    await acts.mergeChunks({
-      bucket: sourceFile!.bucket,
-      fileReferences: errorFileReferences,
-      outputFileReference: "errors.json",
-    });
+    const validationsFileReferences = await Promise.all(parallelValidations);
     await acts.mergeChunks({
       bucket: sourceFile!.bucket,
-      fileReferences: chunkedFileReferences,
-      outputFileReference: "target.json",
+      fileReferences: validationsFileReferences,
+      outputFileReference: "validations.json",
     });
     console.log(`all validations took ${Date.now() - startAllValidations}ms`);
+    latestValidations = {
+      bucket: sourceFile!.bucket,
+      fileReference: "validations.json",
+    };
     isValidating = false;
   }
 }

From c58b0474526d5ed3b34b84c133831d0e484f207e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 15:30:22 +0100
Subject: [PATCH 06/13] feat: add enum validator and refactor validation a bit

---
 app/temporal/src/activities.ts                |  14 +-
 app/temporal/src/domain/ColumnValidation.ts   |   9 +-
 app/temporal/src/domain/DataAnalyzer.spec.ts  | 138 +++++++++++++-----
 app/temporal/src/domain/DataAnalyzer.ts       |  22 ++-
 .../src/domain/validators/EmailValidator.ts   |   6 +-
 .../src/domain/validators/EnumValidator.ts    |  27 ++++
 .../src/domain/validators/RegexValidator.ts   |  24 +--
 .../src/domain/validators/UniqueValidator.ts  |  22 ++-
 app/temporal/src/domain/validators/index.ts   |  15 +-
 .../src/workflows/importer.workflow.ts        |  23 ++-
 10 files changed, 210 insertions(+), 90 deletions(-)
 create mode 100644 app/temporal/src/domain/validators/EnumValidator.ts

diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts
index a38d8c2..c26fd1c 100644
--- a/app/temporal/src/activities.ts
+++ b/app/temporal/src/activities.ts
@@ -3,8 +3,11 @@ import csv from "csv";
 import { chunk } from "lodash";
 import XLSX from "xlsx";
 import { ColumnConfig } from "./domain/ColumnConfig";
-import { DataAnalyzer, DataMappingRecommendation } from "./domain/DataAnalyzer";
-import { ValidatorType } from "./domain/validators";
+import {
+  ColumnValidators,
+  DataAnalyzer,
+  DataMappingRecommendation,
+} from "./domain/DataAnalyzer";
 import { FileStore } from "./infrastructure/FileStore";
 import { Mapping } from "./workflows/importer.workflow";
 export interface DownloadSourceFileParams {
@@ -17,11 +20,6 @@ export interface DownloadSourceFileReturnType {
   localFilePath: string;
 }
 
-export type ValidatorColumns = Record<
-  ValidatorType,
-  { column: string; regex?: string | undefined }[]
->;
-
 export function makeActivities(
   fileStore: FileStore,
   dataAnalyzer: DataAnalyzer
@@ -148,7 +146,7 @@ export function makeActivities(
       bucket: string;
       fileReference: string;
       statsFileReference: string;
-      validatorColumns: ValidatorColumns;
+      validatorColumns: ColumnValidators;
     }) => {
       const referenceId = params.fileReference.split("-")[1].split(".")[0];
       const fileData = await fileStore.getFile(
diff --git a/app/temporal/src/domain/ColumnValidation.ts b/app/temporal/src/domain/ColumnValidation.ts
index 9610b1e..23b6a7a 100644
--- a/app/temporal/src/domain/ColumnValidation.ts
+++ b/app/temporal/src/domain/ColumnValidation.ts
@@ -2,5 +2,12 @@ import { ValidatorType } from "./validators";
 
 export interface ColumnValidation {
   type: ValidatorType;
-  regex?: string;
+}
+
+export interface RegexColumnValidation extends ColumnValidation {
+  regex: string;
+}
+
+export interface EnumerationColumnValidation extends ColumnValidation {
+  values: string[];
 }
diff --git a/app/temporal/src/domain/DataAnalyzer.spec.ts b/app/temporal/src/domain/DataAnalyzer.spec.ts
index 5f586a8..0b23a42 100644
--- a/app/temporal/src/domain/DataAnalyzer.spec.ts
+++ b/app/temporal/src/domain/DataAnalyzer.spec.ts
@@ -1,6 +1,13 @@
 import { ColumnConfig } from "./ColumnConfig";
-import { DataAnalyzer, Stats } from "./DataAnalyzer";
-import { ValidatorType } from "./validators";
+import {
+  EnumerationColumnValidation,
+  RegexColumnValidation,
+} from "./ColumnValidation";
+import {
+  ColumnValidators,
+  DataAnalyzer,
+  SourceFileStatsPerColumn,
+} from "./DataAnalyzer";
 
 describe("DataAnalyzer", () => {
   const analyzer = new DataAnalyzer();
@@ -94,7 +101,7 @@ describe("DataAnalyzer", () => {
 
   describe("validation", () => {
     it("should validate required columns", () => {
-      const rowsWithMissingName = [
+      const rowsWithMissingName: Record<string, string | number | null>[] = [
         {
           __rowId: 0,
           name: "John",
@@ -103,11 +110,10 @@ describe("DataAnalyzer", () => {
         { __rowId: 2, age: 25 },
         { __rowId: 3, name: "" },
         { __rowId: 4, name: null },
-        { __rowId: 5, name: undefined },
       ];
       const validatorColumns = {
-        required: [{ column: "name" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
+        required: [{ column: "name", config: { type: "required" } }],
+      } as ColumnValidators;
       const stats = {};
       const result = analyzer.processDataValidations(
         rowsWithMissingName,
@@ -155,16 +161,6 @@ describe("DataAnalyzer", () => {
             },
           ],
         },
-        {
-          rowId: 5,
-          column: "name",
-          errors: [
-            {
-              message: "value is required",
-              type: "required",
-            },
-          ],
-        },
       ]);
     });
 
@@ -176,14 +172,16 @@ describe("DataAnalyzer", () => {
         },
         { __rowId: 1, name: "John" },
         { __rowId: 2, name: "Egon" },
-        { __rowId: 3 },
-        { __rowId: 4 },
+        { __rowId: 3, name: "" },
+        { __rowId: 4, name: "" },
         { __rowId: 5, name: "John" },
       ];
       const validatorColumns = {
-        unique: [{ column: "name" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
-      const stats: Stats = { name: { nonunique: { John: 3, undefined: 2 } } };
+        unique: [{ column: "name", config: { type: "unique" } }],
+      } as ColumnValidators;
+      const stats: SourceFileStatsPerColumn = {
+        name: { nonunique: { John: 3, "": 2 } },
+      };
       const result = analyzer.processDataValidations(
         rowsWithDuplicateValues,
         validatorColumns,
@@ -251,12 +249,20 @@ describe("DataAnalyzer", () => {
         },
         { __rowId: 1, Postleitzahl: "90596" },
         { __rowId: 2, Postleitzahl: "x90596" },
-        { __rowId: 3 },
+        { __rowId: 3, Postleitzahl: "" },
         { __rowId: 4, Postleitzahl: "123" },
       ];
       const validatorColumns = {
-        regex: [{ column: "Postleitzahl", regex: "^[0-9]{5}$" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
+        regex: [
+          {
+            column: "Postleitzahl",
+            config: {
+              type: "regex",
+              regex: "^[0-9]{5}$",
+            } as RegexColumnValidation,
+          },
+        ],
+      } as ColumnValidators;
       const stats = {};
       const result = analyzer.processDataValidations(
         rowsWithRegexValues,
@@ -309,12 +315,12 @@ describe("DataAnalyzer", () => {
         },
         { __rowId: 2, phone: "+49 151/40604777 " },
         { __rowId: 3, phone: "foo" },
-        { __rowId: 4 },
+        { __rowId: 4, phone: "" },
       ];
 
       const validatorColumns = {
-        phone: [{ column: "phone" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
+        phone: [{ column: "phone", config: { type: "phone" } }],
+      } as ColumnValidators;
       const stats = {};
       const result = analyzer.processDataValidations(
         rowsWithPhoneValues,
@@ -352,11 +358,11 @@ describe("DataAnalyzer", () => {
         { __rowId: 2, email: "fiedlefl@gmail" },
         { __rowId: 3, email: "fiedlefl@gmail@test.com" },
         { __rowId: 4, email: "foo" },
-        { __rowId: 5 },
+        { __rowId: 5, email: "" },
       ];
       const validatorColumns = {
-        email: [{ column: "email" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
+        email: [{ column: "email", config: { type: "email" } }],
+      } as ColumnValidators;
       const stats = {};
       const result = analyzer.processDataValidations(
         rowsWithEmailValues,
@@ -410,13 +416,23 @@ describe("DataAnalyzer", () => {
     it("should validate multiple validations", () => {
       const rowsWithDuplicateValues = [{ __rowId: 0 }, { __rowId: 1 }];
       const validatorColumns = {
-        required: [{ column: "name" }],
-        unique: [{ column: "name" }],
-        phone: [{ column: "name" }],
-        email: [{ column: "name" }],
-        regex: [{ column: "name", regex: "^[0-9]{5}$" }],
-      } as Record<ValidatorType, { column: string; regex?: string }[]>;
-      const stats: Stats = { name: { nonunique: { undefined: 2 } } };
+        required: [{ column: "name", config: { type: "required" } }],
+        unique: [{ column: "name", config: { type: "unique" } }],
+        phone: [{ column: "name", config: { type: "phone" } }],
+        email: [{ column: "name", config: { type: "email" } }],
+        regex: [
+          {
+            column: "name",
+            config: {
+              type: "regex",
+              regex: "^[0-9]{5}$",
+            } as RegexColumnValidation,
+          },
+        ],
+      } as ColumnValidators;
+      const stats: SourceFileStatsPerColumn = {
+        name: { nonunique: { undefined: 2 } },
+      };
       const result = analyzer.processDataValidations(
         rowsWithDuplicateValues,
         validatorColumns,
@@ -479,6 +495,54 @@ describe("DataAnalyzer", () => {
         },
       ]);
     });
+
+    it("should validate enum values", () => {
+      const rowsWithEmailValues = [
+        { __rowId: 0, department: "Department 1" },
+        { __rowId: 1, department: "Department 2" },
+        { __rowId: 2, department: "Department 3" },
+        { __rowId: 3, department: "" },
+      ];
+      const validatorColumns = {
+        enum: [
+          {
+            column: "department",
+            config: {
+              type: "enum",
+              values: ["Department 1", "Department 2"],
+            } as EnumerationColumnValidation,
+          },
+        ],
+      } as ColumnValidators;
+      const stats = {};
+      const result = analyzer.processDataValidations(
+        rowsWithEmailValues,
+        validatorColumns,
+        stats
+      );
+      expect(result).toEqual([
+        {
+          column: "department",
+          rowId: 2,
+          errors: [
+            {
+              message: "value is not a valid enum",
+              type: "enum",
+            },
+          ],
+        },
+        {
+          column: "department",
+          errors: [
+            {
+              message: "value is not a valid enum",
+              type: "enum",
+            },
+          ],
+          rowId: 3,
+        },
+      ]);
+    });
   });
 
   it("should return stats", () => {
diff --git a/app/temporal/src/domain/DataAnalyzer.ts b/app/temporal/src/domain/DataAnalyzer.ts
index a0ad60b..474e647 100644
--- a/app/temporal/src/domain/DataAnalyzer.ts
+++ b/app/temporal/src/domain/DataAnalyzer.ts
@@ -1,6 +1,6 @@
 import Fuse from "fuse.js";
-import type { ValidatorColumns } from "../activities";
 import { ColumnConfig } from "./ColumnConfig";
+import { ColumnValidation } from "./ColumnValidation";
 import { ValidationError } from "./ValidationError";
 import { ValidatorType, validators } from "./validators";
 
@@ -10,7 +10,15 @@ export interface DataMappingRecommendation {
   confidence: number;
 }
 
-export type Stats = Record<string, { nonunique: Record<string, number> }>;
+export type SourceFileStatsPerColumn = Record<
+  string,
+  { nonunique: Record<string, number> }
+>;
+
+export type ColumnValidators = Record<
+  ValidatorType,
+  { column: string; config: ColumnValidation }[]
+>;
 
 export class DataAnalyzer {
   constructor() {}
@@ -58,9 +66,9 @@ export class DataAnalyzer {
   }
 
   public processDataValidations(
-    data: Record<string, unknown>[],
-    validatorColumns: ValidatorColumns,
-    stats: Stats
+    data: Record<string, string | number | null>[],
+    validatorColumns: ColumnValidators,
+    stats: SourceFileStatsPerColumn
   ): { rowId: number; column: string; errors: ValidationError[] }[] {
     const chunkErrors: {
       rowId: number;
@@ -101,9 +109,9 @@ export class DataAnalyzer {
   public getStats(
     data: Record<string, unknown>[],
     columnsToVerify: string[]
-  ): Stats {
+  ): SourceFileStatsPerColumn {
     // nonunique
-    const stats = {} as Stats;
+    const stats = {} as SourceFileStatsPerColumn;
     for (const column of columnsToVerify) {
       const duplicates = new Map();
       data.forEach((row) => {
diff --git a/app/temporal/src/domain/validators/EmailValidator.ts b/app/temporal/src/domain/validators/EmailValidator.ts
index 0b9e43a..57c92d3 100644
--- a/app/temporal/src/domain/validators/EmailValidator.ts
+++ b/app/temporal/src/domain/validators/EmailValidator.ts
@@ -1,12 +1,14 @@
+import { Validator } from ".";
+import { ColumnValidation } from "../ColumnValidation";
 import { ValidationError } from "../ValidationError";
 
 const EMAIL_REGEX =
   /^[-!#$%&'*+\/0-9=?A-Z^_a-z`{|}~](\.?[-!#$%&'*+\/0-9=?A-Z^_a-z`{|}~])*@[a-zA-Z0-9](-*\.?[a-zA-Z0-9])*\.[a-zA-Z](-?[a-zA-Z0-9])+$/;
 
-export class EmailValidator {
+export class EmailValidator implements Validator {
   validate(
     row: Record<string, unknown>,
-    columnConfig: { column: string; regex?: string }[]
+    columnConfig: { column: string; config: ColumnValidation }[]
   ): Record<string, ValidationError> {
     const errors: Record<string, ValidationError> = {};
     const columnsToValidate = columnConfig.map((item) => item.column);
diff --git a/app/temporal/src/domain/validators/EnumValidator.ts b/app/temporal/src/domain/validators/EnumValidator.ts
new file mode 100644
index 0000000..5e4cd44
--- /dev/null
+++ b/app/temporal/src/domain/validators/EnumValidator.ts
@@ -0,0 +1,27 @@
+import { Validator } from ".";
+import { EnumerationColumnValidation } from "../ColumnValidation";
+import { ValidationError } from "../ValidationError";
+
+export class EnumValidator implements Validator {
+  public validate(
+    row: Record<string, string | number | null>,
+    columnConfigs: { column: string; config: EnumerationColumnValidation }[]
+  ): Record<string, ValidationError> {
+    const errors: Record<string, ValidationError> = {};
+    for (const { column, config } of columnConfigs) {
+      let dataToValidate = row[column];
+      if (!dataToValidate || typeof dataToValidate !== "string") {
+        errors[column] = {
+          type: "enum",
+          message: "value is not a valid enum",
+        };
+      } else if (config.values.includes(dataToValidate) === false) {
+        errors[column] = {
+          type: "enum",
+          message: "value is not a valid enum",
+        };
+      }
+    }
+    return errors;
+  }
+}
diff --git a/app/temporal/src/domain/validators/RegexValidator.ts b/app/temporal/src/domain/validators/RegexValidator.ts
index 3b01bc6..da88a74 100644
--- a/app/temporal/src/domain/validators/RegexValidator.ts
+++ b/app/temporal/src/domain/validators/RegexValidator.ts
@@ -1,26 +1,30 @@
+import { Validator } from ".";
+import { RegexColumnValidation } from "../ColumnValidation";
 import { ValidationError } from "../ValidationError";
 
-export class RegexValidator {
+export class RegexValidator implements Validator {
   validate(
-    row: Record<string, unknown>,
-    columnConfig: { column: string; regex?: string }[]
+    row: Record<string, string | number | null>,
+    columnConfig: { column: string; config: RegexColumnValidation }[]
   ): Record<string, ValidationError> {
     const errors: Record<string, ValidationError> = {};
     const cache: Record<string, boolean> = {};
-    for (const config of columnConfig) {
-      const { column: columnToValidate, regex } = config;
-      let dataToValidate = row[columnToValidate];
-      if (cache[columnToValidate] === false) {
-        errors[columnToValidate] = {
+    for (const {
+      column,
+      config: { regex },
+    } of columnConfig) {
+      let dataToValidate = row[column];
+      if (cache[column] === false) {
+        errors[column] = {
           type: "regex",
           message: `value does not match regex ${regex}`,
         };
       } else {
         const regexTestResult =
           regex && new RegExp(regex).test(dataToValidate as string);
-        cache[columnToValidate] = regexTestResult as boolean;
+        cache[column] = regexTestResult as boolean;
         if (regexTestResult === false) {
-          errors[columnToValidate] = {
+          errors[column] = {
             type: "regex",
             message: `value does not match regex ${regex}`,
           };
diff --git a/app/temporal/src/domain/validators/UniqueValidator.ts b/app/temporal/src/domain/validators/UniqueValidator.ts
index 78931f4..dfaef9f 100644
--- a/app/temporal/src/domain/validators/UniqueValidator.ts
+++ b/app/temporal/src/domain/validators/UniqueValidator.ts
@@ -1,23 +1,21 @@
-import { Stats } from "../DataAnalyzer";
+import { Validator } from ".";
+import { ColumnValidation } from "../ColumnValidation";
+import { SourceFileStatsPerColumn } from "../DataAnalyzer";
 import { ValidationError } from "../ValidationError";
 
-export class UniqueValidator {
+export class UniqueValidator implements Validator {
   constructor() {}
 
   validate(
     row: Record<string, unknown>,
-    columnConfig: { column: string; regex?: string }[],
-    stats: Stats = {}
+    columnConfig: { column: string; config: ColumnValidation }[],
+    stats: SourceFileStatsPerColumn = {}
   ) {
     const errors: Record<string, ValidationError> = {};
-    const columnsToValidate = columnConfig.map((item) => item.column);
-    for (const columnToValidate of columnsToValidate) {
-      let dataToValidate = row[columnToValidate];
-      if (
-        stats[columnToValidate] &&
-        stats[columnToValidate].nonunique[dataToValidate as string]
-      ) {
-        errors[columnToValidate] = {
+    for (const { column } of columnConfig) {
+      let dataToValidate = row[column];
+      if (stats[column] && stats[column].nonunique[dataToValidate as string]) {
+        errors[column] = {
           type: "unique",
           message: "value is not unique",
         };
diff --git a/app/temporal/src/domain/validators/index.ts b/app/temporal/src/domain/validators/index.ts
index 215e603..a74c1fa 100644
--- a/app/temporal/src/domain/validators/index.ts
+++ b/app/temporal/src/domain/validators/index.ts
@@ -1,15 +1,28 @@
+import { ColumnValidation } from "../ColumnValidation";
+import { SourceFileStatsPerColumn } from "../DataAnalyzer";
+import { ValidationError } from "../ValidationError";
 import { EmailValidator } from "./EmailValidator";
+import { EnumValidator } from "./EnumValidator";
 import { PhoneValidator } from "./PhoneValidator";
 import { RegexValidator } from "./RegexValidator";
 import { RequiredValidator } from "./RequiredValidator";
 import { UniqueValidator } from "./UniqueValidator";
 
-export const validators = {
+export const validators: Record<string, Validator> = {
   unique: new UniqueValidator(),
   required: new RequiredValidator(),
   regex: new RegexValidator(),
   phone: new PhoneValidator(),
   email: new EmailValidator(),
+  enum: new EnumValidator(),
 };
 
 export type ValidatorType = keyof typeof validators;
+
+export interface Validator {
+  validate(
+    row: Record<string, string | number | null>,
+    columnConfigs: { column: string; config: ColumnValidation }[],
+    stats: SourceFileStatsPerColumn
+  ): Record<string, ValidationError>;
+}
diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index 5960da7..d161a40 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -10,9 +10,12 @@ import {
   setHandler,
 } from "@temporalio/workflow";
 import pLimit from "p-limit";
-import { ValidatorColumns, makeActivities } from "../activities";
+import { makeActivities } from "../activities";
 import { ColumnConfig } from "../domain/ColumnConfig";
-import { DataMappingRecommendation } from "../domain/DataAnalyzer";
+import {
+  ColumnValidators,
+  DataMappingRecommendation,
+} from "../domain/DataAnalyzer";
 import { DataSetPatch } from "../domain/DataSet";
 export interface ImporterWorkflowParams {
   name: string;
@@ -243,20 +246,16 @@ export async function importer(params: ImporterWorkflowParams) {
       (column) => column.validations?.length
     );
 
-    const validatorColumns = {} as ValidatorColumns;
+    const validatorColumns = {} as ColumnValidators;
     for (const column of allColumnsWithValidators) {
       for (const validator of column.validations!) {
-        if (validatorColumns[validator.type] === undefined) {
+        if (!validatorColumns[validator.type]) {
           validatorColumns[validator.type] = [];
         }
-        if (validator.type === "regex") {
-          validatorColumns[validator.type].push({
-            column: column.key,
-            regex: validator.regex,
-          });
-        } else {
-          validatorColumns[validator.type].push({ column: column.key });
-        }
+        validatorColumns[validator.type].push({
+          column: column.key,
+          config: validator,
+        });
       }
     }
 

From 752f1f6061690b6e1e95e518ca9cc517762a6a0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 15:56:22 +0100
Subject: [PATCH 07/13] fix: wrong key for validations and wrong mappings

---
 app/src/app/CreateTestImport.tsx              | 19 ++++++++++++
 app/temporal/src/activities.ts                | 31 +++++++++++--------
 .../src/workflows/importer.workflow.ts        | 11 +++++--
 3 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/app/src/app/CreateTestImport.tsx b/app/src/app/CreateTestImport.tsx
index fc3dea5..0a60645 100644
--- a/app/src/app/CreateTestImport.tsx
+++ b/app/src/app/CreateTestImport.tsx
@@ -20,6 +20,12 @@ export const CreateTestImporter = () => {
             key: "email",
             label: "E-Mail",
             type: "text",
+            validations: [
+              {
+                type: "email",
+              },
+              { type: "unique" },
+            ],
           },
           {
             key: "work role",
@@ -27,6 +33,19 @@ export const CreateTestImporter = () => {
             label: "Role",
             type: "text",
           },
+          {
+            key: "department",
+            label: "Department",
+            keyAlternatives: ["abteilung"],
+            type: "text",
+            validations: [
+              {
+                type: "enum",
+
+                values: ["IT", "HR", "Support"],
+              },
+            ],
+          },
         ],
       }),
     });
diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts
index c26fd1c..586f652 100644
--- a/app/temporal/src/activities.ts
+++ b/app/temporal/src/activities.ts
@@ -1,6 +1,6 @@
 import { ApplicationFailure } from "@temporalio/workflow";
 import csv from "csv";
-import { chunk } from "lodash";
+import { chunk, pull } from "lodash";
 import XLSX from "xlsx";
 import { ColumnConfig } from "./domain/ColumnConfig";
 import {
@@ -102,26 +102,31 @@ export function makeActivities(
         params.bucket,
         params.fileReference
       );
-      const jsonData: Record<string, unknown>[] = JSON.parse(
+      const sourceJsonData: Record<string, unknown>[] = JSON.parse(
         fileData.toString()
       );
-      const mappedData = jsonData.map((row) => {
+      const mappedData = sourceJsonData.map((row) => {
         const newRow: Record<string, unknown> = {};
         newRow.__rowId = row.__rowId;
-        for (const mapping of params.dataMapping.filter(
-          (mapping) => mapping.targetColumn
-        )) {
+        const mappingsWithSourceColumn = params.dataMapping.filter(
+          (mapping) => mapping.sourceColumn
+        );
+        for (const mapping of mappingsWithSourceColumn) {
           newRow[mapping.targetColumn as string] = row[mapping.sourceColumn!];
         }
         return newRow;
       });
-
+      const mappedDataChunks = chunk(mappedData, 5000);
       return await Promise.all(
-        chunk(mappedData, 5000).map(async (json, index) => {
-          const jsonData = Buffer.from(JSON.stringify(json));
-          const chunktFileReference = `mapped-${index}.json`;
-          await fileStore.putFile(params.bucket, chunktFileReference, jsonData);
-          return chunktFileReference;
+        mappedDataChunks.map(async (json, index) => {
+          const mappedChunkJsonData = Buffer.from(JSON.stringify(json));
+          const chunkedFileReference = `mapped-${index}.json`;
+          await fileStore.putFile(
+            params.bucket,
+            chunkedFileReference,
+            mappedChunkJsonData
+          );
+          return chunkedFileReference;
         })
       );
     },
@@ -136,7 +141,7 @@ export function makeActivities(
       );
       const jsonData = JSON.parse(fileData.toString());
       // all rows should have all available headers (see source file processing)
-      const sourceColumns = Object.keys(jsonData[0]);
+      const sourceColumns = pull(Object.keys(jsonData[0]), "__rowId");
       return dataAnalyzer.generateMappingRecommendations(
         sourceColumns,
         params.columnConfig
diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index d161a40..d561f4e 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -189,12 +189,17 @@ export async function importer(params: ImporterWorkflowParams) {
       columnConfig: params.columnConfig,
     });
 
-    await condition(() => configuredMappings !== null, startImportTimeout);
-
+    const hasConfiguredMappings = await condition(
+      () => configuredMappings !== null,
+      startImportTimeout
+    );
+    if (!hasConfiguredMappings) {
+      throw ApplicationFailure.nonRetryable("Timeout: mappings not configured");
+    }
     const chunkedFileReferences = await acts.applyMappings({
       bucket: sourceFile!.bucket,
       fileReference: sourceFileReference,
-      dataMapping: dataMappingRecommendations.map((item) => ({
+      dataMapping: configuredMappings!.map((item) => ({
         sourceColumn: item.sourceColumn,
         targetColumn: item.targetColumn,
       })),

From 21f866f7cb7c4faa60e1bf42a5d122ee695743c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 16:08:26 +0100
Subject: [PATCH 08/13] fix: only map selected target columns

---
 app/temporal/src/activities.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts
index 586f652..c42ce88 100644
--- a/app/temporal/src/activities.ts
+++ b/app/temporal/src/activities.ts
@@ -108,10 +108,10 @@ export function makeActivities(
       const mappedData = sourceJsonData.map((row) => {
         const newRow: Record<string, unknown> = {};
         newRow.__rowId = row.__rowId;
-        const mappingsWithSourceColumn = params.dataMapping.filter(
-          (mapping) => mapping.sourceColumn
+        const mappingsWithTargetColumn = params.dataMapping.filter(
+          (mapping) => mapping.targetColumn
         );
-        for (const mapping of mappingsWithSourceColumn) {
+        for (const mapping of mappingsWithTargetColumn) {
           newRow[mapping.targetColumn as string] = row[mapping.sourceColumn!];
         }
         return newRow;

From 8e4ac7b0b015f1ee46afa601fc02950ed8b76935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Wed, 24 Jan 2024 16:47:49 +0100
Subject: [PATCH 09/13] feat: show validation errors in table (very basic)

---
 app/package-lock.json                         |  59 ++++++++
 app/package.json                              |   3 +-
 .../app/importer/[id]/validate/Validation.tsx |   1 -
 .../[id]/validate/ValidationTable.tsx         | 129 ++++++++++++------
 app/src/components/ui/badge.tsx               |  36 +++++
 app/src/components/ui/tooltip.tsx             |  30 ++++
 app/temporal/src/activities.ts                |   6 +-
 .../src/workflows/importer.workflow.ts        |   5 -
 8 files changed, 215 insertions(+), 54 deletions(-)
 create mode 100644 app/src/components/ui/badge.tsx
 create mode 100644 app/src/components/ui/tooltip.tsx

diff --git a/app/package-lock.json b/app/package-lock.json
index 52ec471..4718c78 100644
--- a/app/package-lock.json
+++ b/app/package-lock.json
@@ -11,6 +11,7 @@
         "@faker-js/faker": "^8.3.1",
         "@radix-ui/react-select": "^2.0.0",
         "@radix-ui/react-slot": "^1.0.2",
+        "@radix-ui/react-tooltip": "^1.0.7",
         "@tanstack/react-table": "^8.11.7",
         "@tanstack/react-virtual": "^3.0.2",
         "@temporalio/activity": "^1.9.0",
@@ -1874,6 +1875,30 @@
         }
       }
     },
+    "node_modules/@radix-ui/react-presence": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@radix-ui/react-primitive": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
@@ -1958,6 +1983,40 @@
         }
       }
     },
+    "node_modules/@radix-ui/react-tooltip": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
+      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-popper": "1.1.3",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "@radix-ui/react-visually-hidden": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@radix-ui/react-use-callback-ref": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
diff --git a/app/package.json b/app/package.json
index 0de92c3..e93ba08 100644
--- a/app/package.json
+++ b/app/package.json
@@ -19,6 +19,7 @@
     "@faker-js/faker": "^8.3.1",
     "@radix-ui/react-select": "^2.0.0",
     "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-tooltip": "^1.0.7",
     "@tanstack/react-table": "^8.11.7",
     "@tanstack/react-virtual": "^3.0.2",
     "@temporalio/activity": "^1.9.0",
@@ -31,8 +32,8 @@
     "dotenv": "^16.3.2",
     "env-var": "^7.4.1",
     "fuse.js": "^6.6.2",
-    "libphonenumber-js": "^1.10.54",
     "immer": "^10.0.3",
+    "libphonenumber-js": "^1.10.54",
     "lodash": "^4.17.21",
     "lucide-react": "^0.312.0",
     "minio": "^7.1.3",
diff --git a/app/src/app/importer/[id]/validate/Validation.tsx b/app/src/app/importer/[id]/validate/Validation.tsx
index 32632d6..95db22d 100644
--- a/app/src/app/importer/[id]/validate/Validation.tsx
+++ b/app/src/app/importer/[id]/validate/Validation.tsx
@@ -45,7 +45,6 @@ const Validation = ({
     undefined,
     initialValidation
   );
-  console.log(sourceData, patches, validations);
   return (
     <div>
       <h1 className="text-3xl font-bold mb-4">Validate your data</h1>
diff --git a/app/src/app/importer/[id]/validate/ValidationTable.tsx b/app/src/app/importer/[id]/validate/ValidationTable.tsx
index f750972..e574f33 100644
--- a/app/src/app/importer/[id]/validate/ValidationTable.tsx
+++ b/app/src/app/importer/[id]/validate/ValidationTable.tsx
@@ -5,6 +5,7 @@ import {
   ImporterDto,
   SourceData,
 } from "@/app/api/importer/[slug]/ImporterDto";
+import { Badge } from "@/components/ui/badge";
 import {
   Table,
   TableBody,
@@ -13,6 +14,8 @@ import {
   TableHeader,
   TableRow,
 } from "@/components/ui/table";
+import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
+import { cn } from "@/lib/utils";
 import {
   ColumnDef,
   Row,
@@ -21,6 +24,7 @@ import {
   useReactTable,
 } from "@tanstack/react-table";
 import { useVirtualizer } from "@tanstack/react-virtual";
+import { groupBy } from "lodash";
 import React from "react";
 
 type Props = {
@@ -32,6 +36,8 @@ type Props = {
 
 const ValidationTable = (props: Props) => {
   const tableContainerRef = React.useRef<HTMLDivElement>(null);
+  // TODO use helper to define columns and headers?
+  // https://tanstack.com/table/v8/docs/guide/column-defs
   const columns: ColumnDef<any>[] = React.useMemo(
     () =>
       props.importerDto.config.columnConfig.map(
@@ -48,6 +54,11 @@ const ValidationTable = (props: Props) => {
     // TODO merge with patches
     return props.data;
   }, [props.data]);
+
+  const validationsByColumn = React.useMemo(() => {
+    return groupBy(props.validations, "column");
+  }, [props.validations]);
+
   const table = useReactTable({
     data,
     columns,
@@ -73,55 +84,83 @@ const ValidationTable = (props: Props) => {
       className="rounded-md overflow-auto border h-[80vh]"
       ref={tableContainerRef}
     >
-      <Table className="relative">
-        <TableHeader className="sticky top-0 z-10 bg-white">
-          {table.getHeaderGroups().map((headerGroup) => (
-            <TableRow key={headerGroup.id} className="flex w-full">
-              {headerGroup.headers.map((header) => {
-                return (
-                  <TableHead
-                    key={header.id}
-                    style={{ width: header.getSize() }}
-                  >
-                    {header.isPlaceholder
-                      ? null
-                      : flexRender(
-                          header.column.columnDef.header,
-                          header.getContext()
+      <TooltipProvider>
+        <Tooltip>
+          <Table className="relative">
+            <TableHeader className="sticky top-0 z-10 bg-white">
+              {table.getHeaderGroups().map((headerGroup) => (
+                <TableRow key={headerGroup.id} className="flex w-full">
+                  {headerGroup.headers.map((header) => {
+                    const numberOfValidations =
+                      validationsByColumn[header.id]?.length ?? 0;
+                    return (
+                      <TableHead
+                        key={header.id}
+                        style={{ width: header.getSize() }}
+                      >
+                        {header.isPlaceholder
+                          ? null
+                          : flexRender(
+                              header.column.columnDef.header,
+                              header.getContext()
+                            )}
+                        {numberOfValidations > 0 && (
+                          <Badge className="ml-4 bg-red-500">
+                            {numberOfValidations} errors
+                          </Badge>
                         )}
-                  </TableHead>
-                );
-              })}
-            </TableRow>
-          ))}
-        </TableHeader>
-        <TableBody style={{ height: `${rowVirtualizer.getTotalSize()}px` }}>
-          {rowVirtualizer.getVirtualItems().map((virtualRow) => {
-            const row = rows[virtualRow.index] as Row<any>;
-            return (
-              <TableRow
-                className="absolute w-full flex"
-                style={{
-                  transform: `translateY(${virtualRow.start}px)`,
-                }}
-                key={row.id}
-                data-state={row.getIsSelected() && "selected"}
-              >
-                {row.getVisibleCells().map((cell) => (
-                  <TableCell
-                    key={cell.id}
+                      </TableHead>
+                    );
+                  })}
+                </TableRow>
+              ))}
+            </TableHeader>
+            <TableBody style={{ height: `${rowVirtualizer.getTotalSize()}px` }}>
+              {rowVirtualizer.getVirtualItems().map((virtualRow) => {
+                const row = rows[virtualRow.index] as Row<any>;
+                return (
+                  <TableRow
+                    className="absolute w-full flex"
                     style={{
-                      width: cell.column.getSize(),
+                      transform: `translateY(${virtualRow.start}px)`,
                     }}
+                    key={row.id}
+                    data-state={row.getIsSelected() && "selected"}
                   >
-                    {flexRender(cell.column.columnDef.cell, cell.getContext())}
-                  </TableCell>
-                ))}
-              </TableRow>
-            );
-          })}
-        </TableBody>
-      </Table>
+                    {row.getVisibleCells().map((cell) => {
+                      const validationsForCell = props.validations.filter(
+                        (validation) =>
+                          validation.rowId === parseInt(row.id) &&
+                          validation.column === cell.column.id
+                      );
+                      const allErrors = validationsForCell.flatMap(
+                        (validation) => validation.errors
+                      );
+                      const hasErrors = allErrors.length > 0;
+                      return (
+                        <TableCell
+                          key={cell.id}
+                          style={{
+                            width: cell.column.getSize(),
+                          }}
+                          className={cn({
+                            "text-red-500 bg-red-200": hasErrors,
+                          })}
+                        >
+                          {flexRender(cell.column.columnDef.cell, {
+                            ...cell.getContext(),
+                            allErrors,
+                          })}
+                        </TableCell>
+                      );
+                    })}
+                  </TableRow>
+                );
+              })}
+            </TableBody>
+          </Table>
+        </Tooltip>
+      </TooltipProvider>
     </div>
   );
 };
diff --git a/app/src/components/ui/badge.tsx b/app/src/components/ui/badge.tsx
new file mode 100644
index 0000000..f000e3e
--- /dev/null
+++ b/app/src/components/ui/badge.tsx
@@ -0,0 +1,36 @@
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const badgeVariants = cva(
+  "inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
+  {
+    variants: {
+      variant: {
+        default:
+          "border-transparent bg-primary text-primary-foreground hover:bg-primary/80",
+        secondary:
+          "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        destructive:
+          "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80",
+        outline: "text-foreground",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+export interface BadgeProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    VariantProps<typeof badgeVariants> {}
+
+function Badge({ className, variant, ...props }: BadgeProps) {
+  return (
+    <div className={cn(badgeVariants({ variant }), className)} {...props} />
+  )
+}
+
+export { Badge, badgeVariants }
diff --git a/app/src/components/ui/tooltip.tsx b/app/src/components/ui/tooltip.tsx
new file mode 100644
index 0000000..30fc44d
--- /dev/null
+++ b/app/src/components/ui/tooltip.tsx
@@ -0,0 +1,30 @@
+"use client"
+
+import * as React from "react"
+import * as TooltipPrimitive from "@radix-ui/react-tooltip"
+
+import { cn } from "@/lib/utils"
+
+const TooltipProvider = TooltipPrimitive.Provider
+
+const Tooltip = TooltipPrimitive.Root
+
+const TooltipTrigger = TooltipPrimitive.Trigger
+
+const TooltipContent = React.forwardRef<
+  React.ElementRef<typeof TooltipPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof TooltipPrimitive.Content>
+>(({ className, sideOffset = 4, ...props }, ref) => (
+  <TooltipPrimitive.Content
+    ref={ref}
+    sideOffset={sideOffset}
+    className={cn(
+      "z-50 overflow-hidden rounded-md border bg-popover px-3 py-1.5 text-sm text-popover-foreground shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
+      className
+    )}
+    {...props}
+  />
+))
+TooltipContent.displayName = TooltipPrimitive.Content.displayName
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }
diff --git a/app/temporal/src/activities.ts b/app/temporal/src/activities.ts
index c42ce88..8c0ac16 100644
--- a/app/temporal/src/activities.ts
+++ b/app/temporal/src/activities.ts
@@ -62,7 +62,6 @@ export function makeActivities(
             }
           );
           console.log("received rows", json.length);
-
           break;
         case "xlsx":
           const workbook = XLSX.read(fileData, { type: "buffer" });
@@ -74,7 +73,6 @@ export function makeActivities(
             defval: "",
           });
           console.log("received rows", json.length);
-
           break;
         default:
           throw ApplicationFailure.nonRetryable(
@@ -153,6 +151,7 @@ export function makeActivities(
       statsFileReference: string;
       validatorColumns: ColumnValidators;
     }) => {
+      console.time("validations");
       const referenceId = params.fileReference.split("-")[1].split(".")[0];
       const fileData = await fileStore.getFile(
         params.bucket,
@@ -176,6 +175,7 @@ export function makeActivities(
         errorFileReference,
         Buffer.from(JSON.stringify(errorData))
       );
+      console.timeEnd("validations");
       return errorFileReference;
     },
     generateStatsFile: async (params: {
@@ -184,6 +184,7 @@ export function makeActivities(
       outputFileReference: string;
       uniqueColumns: string[];
     }): Promise<void> => {
+      console.time("generate-stats");
       const fileData = await fileStore.getFile(
         params.bucket,
         params.fileReference
@@ -198,6 +199,7 @@ export function makeActivities(
         params.outputFileReference,
         statsData
       );
+      console.timeEnd("generate-stats");
     },
     mergeChunks: async (params: {
       bucket: string;
diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index d561f4e..2acf954 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -126,7 +126,6 @@ export async function importer(params: ImporterWorkflowParams) {
   setHandler(
     mappingUpdate,
     (params) => {
-      console.log("handle update mapping", params);
       configuredMappings = params.mappings;
     },
     {
@@ -266,16 +265,13 @@ export async function importer(params: ImporterWorkflowParams) {
 
     // TODO: apply patches
     const statsFileReference = "stats.json";
-    const startStats = Date.now();
     await acts.generateStatsFile({
       bucket: sourceFile!.bucket,
       fileReference: sourceFileReference,
       outputFileReference: statsFileReference,
       uniqueColumns: validatorColumns.unique?.map((item) => item.column) ?? [],
     });
-    console.log(`generate stats file took ${Date.now() - startStats}ms`);
 
-    const startAllValidations = Date.now();
     const limit = pLimit(100);
     const parallelValidations = chunkedFileReferences.map((fileReference) =>
       limit(() =>
@@ -293,7 +289,6 @@ export async function importer(params: ImporterWorkflowParams) {
       fileReferences: validationsFileReferences,
       outputFileReference: "validations.json",
     });
-    console.log(`all validations took ${Date.now() - startAllValidations}ms`);
     latestValidations = {
       bucket: sourceFile!.bucket,
       fileReference: "validations.json",

From 37e9a13d4dc800230023d0728e66fe7156fe2e05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Thu, 25 Jan 2024 08:43:58 +0100
Subject: [PATCH 10/13] chore: add all validation errors

---
 app/src/app/api/importer/[slug]/ImporterDto.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/src/app/api/importer/[slug]/ImporterDto.ts b/app/src/app/api/importer/[slug]/ImporterDto.ts
index f7cbc60..ce49ce7 100644
--- a/app/src/app/api/importer/[slug]/ImporterDto.ts
+++ b/app/src/app/api/importer/[slug]/ImporterDto.ts
@@ -75,7 +75,7 @@ export interface DataValidation {
 }
 
 export interface ValidationError {
-  type: "required" | "unique";
+  type: "required" | "unique" | "regex" | "phone" | "email";
   message: string;
 }
 

From 5988a0de034ca3ef627bd1b4e0c781acb1e03e50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Thu, 25 Jan 2024 08:44:18 +0100
Subject: [PATCH 11/13] chore: refactor if

---
 app/src/app/importer/[id]/validate/page.tsx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/src/app/importer/[id]/validate/page.tsx b/app/src/app/importer/[id]/validate/page.tsx
index fe8fbce..9caa2a4 100644
--- a/app/src/app/importer/[id]/validate/page.tsx
+++ b/app/src/app/importer/[id]/validate/page.tsx
@@ -27,7 +27,9 @@ export default async function page(props: { params: { id: string } }) {
       cache: "no-cache",
     }
   ).then((res) => {
-    if (res.status === 404) return Promise.resolve([]);
+    if (res.status === 404) {
+      return Promise.resolve([]);
+    }
     return res.json() as Promise<DataValidation[]>;
   });
   const initialSourceDataPromise = fetch(

From 231d1526d965337b5dedf9f1d285e1db3587c335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Thu, 25 Jan 2024 08:48:14 +0100
Subject: [PATCH 12/13] chore: add enum validator test to all validations test

---
 app/temporal/src/domain/DataAnalyzer.spec.ts | 26 ++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/app/temporal/src/domain/DataAnalyzer.spec.ts b/app/temporal/src/domain/DataAnalyzer.spec.ts
index 0b23a42..f6c8369 100644
--- a/app/temporal/src/domain/DataAnalyzer.spec.ts
+++ b/app/temporal/src/domain/DataAnalyzer.spec.ts
@@ -413,8 +413,11 @@ describe("DataAnalyzer", () => {
       ]);
     });
 
-    it("should validate multiple validations", () => {
-      const rowsWithDuplicateValues = [{ __rowId: 0 }, { __rowId: 1 }];
+    it("should validate all validations", () => {
+      const rowsWithDuplicateValues = [
+        { __rowId: 0, uniques: "Jan" },
+        { __rowId: 1, uniques: "Flo" },
+      ];
       const validatorColumns = {
         required: [{ column: "name", config: { type: "required" } }],
         unique: [{ column: "name", config: { type: "unique" } }],
@@ -429,6 +432,15 @@ describe("DataAnalyzer", () => {
             } as RegexColumnValidation,
           },
         ],
+        enum: [
+          {
+            column: "uniques",
+            config: {
+              type: "enum",
+              values: ["Jan", "Florian"],
+            } as EnumerationColumnValidation,
+          },
+        ],
       } as ColumnValidators;
       const stats: SourceFileStatsPerColumn = {
         name: { nonunique: { undefined: 2 } },
@@ -493,6 +505,16 @@ describe("DataAnalyzer", () => {
             },
           ],
         },
+        {
+          rowId: 1,
+          column: "uniques",
+          errors: [
+            {
+              message: "value is not a valid enum",
+              type: "enum",
+            },
+          ],
+        },
       ]);
     });
 

From 83c5801bd86ee5f9623b8a369e97a1dc5ee13103 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20K=C3=BChnlein?= <jan.kuehnlein@kultify.com>
Date: Thu, 25 Jan 2024 08:48:32 +0100
Subject: [PATCH 13/13] chore: remove unnecessary map

---
 app/temporal/src/workflows/importer.workflow.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/app/temporal/src/workflows/importer.workflow.ts b/app/temporal/src/workflows/importer.workflow.ts
index 2acf954..f6201a4 100644
--- a/app/temporal/src/workflows/importer.workflow.ts
+++ b/app/temporal/src/workflows/importer.workflow.ts
@@ -198,10 +198,7 @@ export async function importer(params: ImporterWorkflowParams) {
     const chunkedFileReferences = await acts.applyMappings({
       bucket: sourceFile!.bucket,
       fileReference: sourceFileReference,
-      dataMapping: configuredMappings!.map((item) => ({
-        sourceColumn: item.sourceColumn,
-        targetColumn: item.targetColumn,
-      })),
+      dataMapping: configuredMappings!,
     });
     const mappedSourceDataFileReference = "target.json";
     await acts.mergeChunks({