#16 show data in frontend

app/package.json

@@ -19,6 +19,7 @@
     "@faker-js/faker": "^8.3.1",
     "@radix-ui/react-select": "^2.0.0",
     "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-tooltip": "^1.0.7",
     "@tanstack/react-table": "^8.11.7",
     "@tanstack/react-virtual": "^3.0.2",
     "@temporalio/activity": "^1.9.0",
@@ -31,8 +32,8 @@
     "dotenv": "^16.3.2",
     "env-var": "^7.4.1",
     "fuse.js": "^6.6.2",
-    "libphonenumber-js": "^1.10.54",
     "immer": "^10.0.3",
+    "libphonenumber-js": "^1.10.54",
     "lodash": "^4.17.21",
     "lucide-react": "^0.312.0",
     "minio": "^7.1.3",

app/src/app/CreateTestImport.tsx

@@ -20,13 +20,32 @@ export const CreateTestImporter = () => {
             key: "email",
             label: "E-Mail",
             type: "text",
+            validations: [
+              {
+                type: "email",
+              },
+              { type: "unique" },
+            ],
           },
           {
             key: "work role",
             keyAlternatives: ["position"],
             label: "Role",
             type: "text",
           },
+          {
+            key: "department",
+            label: "Department",
+            keyAlternatives: ["abteilung"],
+            type: "text",
+            validations: [
+              {
+                type: "enum",
+
+                values: ["IT", "HR", "Support"],
+              },
+            ],
+          },
         ],
       }),
     });

app/src/app/api/importer/[slug]/ImporterDto.ts

@@ -29,6 +29,13 @@ export interface ImporterStatus {
   isImporting: boolean;
   dataMappingRecommendations: DataMappingRecommendation[] | null;
   dataMapping: DataMapping[] | null;
+
+  sourceData: {
+    bucket: string;
+    fileReference: string;
+  } | null;
+
+  validations: { bucket: string; fileReference: string } | null;
 }
 
 export interface DataMappingRecommendation {
@@ -51,3 +58,27 @@ export interface DataMapping {
   targetColumn: string | null;
   sourceColumn: string;
 }
+
+export interface DataSetPatch {
+  rowId: number;
+  /**
+   * target column
+   */
+  column: string;
+  newValue: string | number | null;
+}
+
+export interface DataValidation {
+  rowId: number;
+  column: string;
+  errors: ValidationError[];
+}
+
+export interface ValidationError {
+  type: "required" | "unique" | "regex" | "phone" | "email";
+  message: string;
+}
+
+export type SourceData = {
+  rowId: number;
+} & Record<string, string | number | null>;

app/src/app/api/importer/[slug]/mappings/route.ts

@@ -1,15 +1,14 @@
 import { getTemporalWorkflowClient } from "@/lib/temporalClient";
 import { NextRequest, NextResponse } from "next/server";
-import { DataMapping } from "../ImporterDto";
 
 export async function PUT(
-  _req: NextRequest,
-  { params, body }: { params: { slug: string }; body: DataMapping[] }
+  req: NextRequest,
+  { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
   const client = getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
-  const mappings = body;
+  const mappings = await req.json();
   await handle.executeUpdate("importer:update-mapping", {
     args: [{ mappings }],
   });

app/src/app/api/importer/[slug]/patches/route.ts

@@ -0,0 +1,14 @@
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { DataSetPatch } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } }
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<DataSetPatch[]>("importer:patches");
+  return NextResponse.json(result);
+}

app/src/app/api/importer/[slug]/source-data/route.ts

@@ -0,0 +1,35 @@
+import minioClient from "@/lib/minioClient";
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { ImporterStatus } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } },
+  res: NextResponse
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<ImporterStatus>("importer:status");
+  if (!result.sourceData) {
+    return new NextResponse(undefined, { status: 404 });
+  }
+  const stream = await minioClient.getObject(
+    result.sourceData.bucket,
+    result.sourceData.fileReference
+  );
+  // TODO find better to directly return stream...
+  const buffers = [];
+  // node.js readable streams implement the async iterator protocol
+  for await (const data of stream) {
+    buffers.push(data);
+  }
+  const fileAsBuffer = Buffer.concat(buffers);
+  return new Response(fileAsBuffer, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+}

app/src/app/api/importer/[slug]/validations/route.ts

@@ -0,0 +1,34 @@
+import minioClient from "@/lib/minioClient";
+import { getTemporalWorkflowClient } from "@/lib/temporalClient";
+import { NextRequest, NextResponse } from "next/server";
+import { ImporterStatus } from "../ImporterDto";
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: { slug: string } }
+) {
+  const { slug: importerId } = params;
+  const client = getTemporalWorkflowClient();
+  const handle = client.getHandle(importerId);
+  const result = await handle.query<ImporterStatus>("importer:status");
+  if (!result.validations) {
+    return new NextResponse("not found", { status: 404 });
+  }
+  const stream = await minioClient.getObject(
+    result.validations.bucket,
+    result.validations.fileReference
+  );
+  // TODO find better to directly return stream...
+  const buffers = [];
+  // node.js readable streams implement the async iterator protocol
+  for await (const data of stream) {
+    buffers.push(data);
+  }
+  const fileAsBuffer = Buffer.concat(buffers);
+  return new Response(fileAsBuffer, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+}

app/src/app/importer/[id]/mapping/ShowMappings.tsx

@@ -34,14 +34,27 @@ interface Mapping {
 
 const ShowMappings = ({ importerDto: initialImporterDto }: Props) => {
   const { push } = useRouter();
+  const [enablePolling, setEnablePolling] = React.useState(false);
   const { importer } = useGetImporter(
     initialImporterDto.importerId,
-    undefined,
+    enablePolling ? 1000 : undefined,
     initialImporterDto
   );
   const {
     status: { dataMappingRecommendations },
   } = importer;
+
+  React.useEffect(() => {
+    if (
+      !dataMappingRecommendations ||
+      dataMappingRecommendations.length === 0
+    ) {
+      setEnablePolling(true);
+    } else {
+      setEnablePolling(false);
+    }
+  }, [dataMappingRecommendations]);
+
   const [currentMappings, setCurrentMappings] = React.useState<Mapping[]>([]);
   React.useEffect(() => {
     const newMapping =

app/src/app/importer/[id]/validate/Validation.tsx

@@ -1,18 +1,60 @@
-import { ImporterDto } from "@/app/api/importer/[slug]/ImporterDto";
+"use client";
+import {
+  DataSetPatch,
+  DataValidation,
+  ImporterDto,
+  SourceData,
+} from "@/app/api/importer/[slug]/ImporterDto";
+import { useGetImporter } from "@/components/hooks/useGetImporter";
+import { useGetPatches } from "@/components/hooks/useGetPatches";
+import { useGetSourceData } from "@/components/hooks/useGetSourceData";
+import { useGetValidations } from "@/components/hooks/useGetValidations";
 import { Button } from "@/components/ui/button";
 import { ChevronRightCircleIcon } from "lucide-react";
 import ValidationTable from "./ValidationTable";
 
 type Props = {
   initialImporterDto: ImporterDto;
+  initialSourceData: SourceData[];
+  initialPatches: DataSetPatch[];
+  initialValidation: DataValidation[];
 };
 
-const Validation = ({ initialImporterDto }: Props) => {
+const Validation = ({
+  initialImporterDto,
+  initialSourceData,
+  initialPatches,
+  initialValidation,
+}: Props) => {
+  const { importer } = useGetImporter(
+    initialImporterDto.importerId,
+    undefined,
+    initialImporterDto
+  );
+  const { data: sourceData } = useGetSourceData(
+    initialImporterDto.importerId,
+    initialSourceData
+  );
+  const { patches } = useGetPatches(
+    initialImporterDto.importerId,
+    undefined,
+    initialPatches
+  );
+  const { validations } = useGetValidations(
+    initialImporterDto.importerId,
+    undefined,
+    initialValidation
+  );
   return (
     <div>
       <h1 className="text-3xl font-bold mb-4">Validate your data</h1>
       <div className="mb-4">
-        <ValidationTable importerDto={initialImporterDto} />
+        <ValidationTable
+          importerDto={importer}
+          data={sourceData}
+          validations={validations}
+          patches={patches}
+        />
       </div>
       <div className="flex justify-end">
         <Button>