chore: add more env vars and docker compose updates

.env.template

@@ -1,3 +1,4 @@
+# INFRASTRUCTURE
 TEMPORAL_VERSION=1.22.4
 TEMPORAL_UI_VERSION=2.21.0
 POSTGRESQL_VERSION=13
@@ -8,4 +9,14 @@ MINIO_ROOT_USER=minioadmin
 MINIO_ROOT_PASSWORD=minioadmin
 MONGO_ROOT_USER=mongoroot
 MONGO_ROOT_PASSWORD=mongoroot
-NEXT_PUBLIC_API_URL=
+
+# HOSTING
+PUBLIC_API_URL=http://localhost:3000
+
+# DEPLOYMENT
+MINIO_HOST=localhost
+MINIO_ACCESS_KEY=
+MINIO_SECRET_KEY=
+MONGO_URL=
+TEMPORAL_ADDRESS=temporal:7233
+TEMPORAL_NAMESPACE=default

app/.env.local.template

@@ -2,4 +2,4 @@ MINIO_HOST=localhost
 MINIO_ACCESS_KEY=
 MINIO_SECRET_KEY=
 MONGO_URL=
-API_URL=http://localhost:3000
+PUBLIC_API_URL=http://localhost:3000

app/src/app/api/importer/[slug]/mappings/recommendations/route.ts

@@ -6,7 +6,7 @@ export async function GET(
   { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const recommendations = await handle.query(
     "importer:data-mapping-recommendations"

app/src/app/api/importer/[slug]/mappings/route.ts

@@ -6,7 +6,7 @@ export async function PUT(
   { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const mappings = await req.json();
   await handle.executeUpdate("importer:update-mapping", {

app/src/app/api/importer/[slug]/records/route.ts

@@ -27,7 +27,7 @@ export async function PATCH(
   { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const updateData = await req.json();
   handle.executeUpdate<void, [{ patches: DataSetPatch[] }]>(

app/src/app/api/importer/[slug]/route.ts

@@ -7,7 +7,7 @@ export async function GET(
   { params }: { params: { slug: string } }
 ) {
   const { slug: importerId } = params;
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   const [status, config] = await Promise.all([
     (await handle.query("importer:status")) as ImporterStatus,

app/src/app/api/importer/route.ts

@@ -14,7 +14,7 @@ export interface ImportWorkflowPostResponse {
 
 export async function POST(req: NextRequest) {
   const importerId = `imp-${randomUUID()}`;
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const body = (await req.json()) as ImportWorkflowPostPayload;
   //TODO validate request
   await client.start("importer", {

app/src/app/api/upload/route.ts

@@ -43,7 +43,7 @@ export async function POST(req: NextRequest) {
     console.error(error);
     return new Response("Failed to upload file", { status: 500 });
   }
-  const client = getTemporalWorkflowClient();
+  const client = await getTemporalWorkflowClient();
   const handle = client.getHandle(importerId);
   await handle.executeUpdate("importer:add-file", {
     args: [

app/src/app/importer/[id]/import/ImportPage.tsx

@@ -34,7 +34,7 @@ const ImportPage = ({ importerDto: initialImporterDto }: Props) => {
         await fetch("/api/upload", { method: "POST", body: formData });
         push("mapping");
       } catch (err) {
-        console.log(err);
+        console.error(err);
       } finally {
         setIsUploading(false);
       }

app/src/app/importer/[id]/mapping/ShowMappings.tsx

@@ -108,7 +108,7 @@ const ShowMappings = ({
       });
       push("validate");
     } catch (err) {
-      console.log(err);
+      console.error(err);
     } finally {
       setIsSavingMapping(false);
     }

app/src/lib/temporalClient.ts

@@ -1,8 +1,18 @@
-import { WorkflowClient } from "@temporalio/client";
-
+import { Connection, WorkflowClient } from "@temporalio/client";
+import env from "env-var";
 export const DEFAULT_TEMPORAL_QUEUE = "imports";
 
-export const getTemporalWorkflowClient = () => {
+let connection: Connection;
+
+export async function getTemporalWorkflowClient() {
   // TODO get options from env
-  return new WorkflowClient({});
-};
+  if (!connection) {
+    connection = await Connection.connect({
+      address: env.get("TEMPORAL_ADDRESS").default("localhost:7233").asString(),
+    });
+  }
+  return new WorkflowClient({
+    connection,
+    namespace: env.get("TEMPORAL_NAMESPACE").default("default").asString(),
+  });
+}

app/temporal/src/activities.ts

@@ -227,7 +227,7 @@ export function makeActivities(
       importerId: string;
       callbackUrl: string;
     }): Promise<void> => {
-      const host = process.env.API_URL ?? "http://localhost:3000";
+      const host = process.env.PUBLIC_API_URL ?? "http://localhost:3000";
       const downloadUrl = `${host}/api/download/${params.importerId}`;
       // we dont await the call
       fetch(params.callbackUrl, {

app/temporal/src/worker.ts

@@ -1,4 +1,4 @@
-import { Worker } from "@temporalio/worker";
+import { NativeConnection, Worker } from "@temporalio/worker";
 import dotenv from "dotenv";
 import env from "env-var";
 import * as Minio from "minio";
@@ -9,8 +9,7 @@ import { Database } from "./infrastructure/Database";
 import { FileStore } from "./infrastructure/FileStore";
 
 dotenv.config({ debug: true });
-console.log(process.env);
-run().catch((err) => console.log(err));
+run().catch((err) => console.error(err));
 
 async function run() {
   const minioClient = new Minio.Client({
@@ -27,7 +26,12 @@ async function run() {
   const database = new Database(mongoClient);
   const dataAnalyzer = new DataAnalyzer();
   const activities = makeActivities(fileStore, database, dataAnalyzer);
+  const nativeConnection = await NativeConnection.connect({
+    address: env.get("TEMPORAL_ADDRESS").default("localhost:7233").asString(),
+  });
   const worker = await Worker.create({
+    connection: nativeConnection,
+    namespace: env.get("TEMPORAL_NAMESPACE").default("default").asString(),
     workflowsPath: require.resolve("./workflows"), // passed to Webpack for bundling
     activities, // directly imported in Node.js
     taskQueue: "imports", // TODO get from env

docker-compose.base.yaml

@@ -6,14 +6,10 @@ services:
     restart: always
     ports:
       - 3000:3000
-    environment:
-      NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL}
   worker:
     image: ghcr.io/jank1310/datadolphin:latest
     command: node /app/worker/worker.js
     restart: always
-    environment:
-      NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL}
   # STORAGE
   minio:
     image: minio/minio:${MINIO_VERSION}

docker-compose.dev.yaml

@@ -0,0 +1,67 @@
+version: "3.7"
+
+services:
+  # STORAGE
+  minio:
+    extends:
+      file: docker-compose.base.yaml
+      service: minio
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
+    volumes:
+      - miniostorage:/data
+  temporal-postgresql:
+    extends:
+      file: docker-compose.base.yaml
+      service: temporal-postgresql
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    networks:
+      - temporal-network
+    volumes:
+      - temporal-postgresql:/var/lib/postgresql/data
+  mongodb:
+    extends:
+      file: docker-compose.base.yaml
+      service: mongodb
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD}
+    volumes:
+      - mongodb:/data/db
+  # TEMPORAL
+  temporal:
+    extends:
+      file: docker-compose.base.yaml
+      service: temporal
+    networks:
+      - temporal-network
+    environment:
+      - DB=postgresql
+      - DB_PORT=5432
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PWD=${POSTGRES_PASSWORD}
+      - POSTGRES_SEEDS=temporal-postgresql
+      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
+  temporal-ui:
+    extends:
+      file: docker-compose.base.yaml
+      service: temporal-ui
+    environment:
+      - TEMPORAL_ADDRESS=temporal:7233
+      - TEMPORAL_CORS_ORIGINS=http://localhost:3000
+    networks:
+      - temporal-network
+networks:
+  temporal-network:
+    driver: bridge
+    name: temporal-network
+  ferretdb-network:
+    driver: bridge
+    name: ferretdb-network
+volumes:
+  miniostorage:
+  temporal-postgresql:
+  mongodb:

docker-compose.yaml

@@ -5,10 +5,21 @@ services:
     extends:
       file: docker-compose.base.yaml
       service: app
+    environment: &defaultEnv
+      NEXT_PUBLIC_API_URL: ${PUBLIC_API_URL}
+      MINIO_HOST: minio
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MONGO_URL: ${MONGO_URL}
+      TEMPORAL_ADDRESS: ${TEMPORAL_ADDRESS}
+      TEMPORAL_NAMESPACE: ${TEMPORAL_NAMESPACE}
   worker-1:
     extends:
       file: docker-compose.base.yaml
       service: worker
+    environment:
+      <<: *defaultEnv
+      PUBLIC_API_URL: ${PUBLIC_API_URL}
   # STORAGE
   minio:
     extends: