first commit

go/sdk/pkg/sdk/ingestion.go

@@ -84,7 +84,7 @@ func (i *Ingestion) IngestFiles(
 		if err != nil {
 			return nil, fmt.Errorf("error marshaling chunking config: %w", err)
 		}
-		writer.WriteField("chunking_settings", string(chunkingConfigJSON))
+		writer.WriteField("chunking_config", string(chunkingConfigJSON))
 	}
 
 	err := writer.Close()
@@ -167,7 +167,7 @@ func (i *Ingestion) UpdateFiles(
 		if err != nil {
 			return nil, fmt.Errorf("error marshaling chunking config: %w", err)
 		}
-		writer.WriteField("chunking_settings", string(chunkingConfigJSON))
+		writer.WriteField("chunking_config", string(chunkingConfigJSON))
 	}
 
 	err = writer.Close()

js/sdk/src/r2rClient.ts

@@ -412,7 +412,7 @@ export class r2rClient {
       metadatas?: Record<string, any>[];
       document_ids?: string[];
       user_ids?: (string | null)[];
-      chunking_settings?: Record<string, any>;
+      chunking_config?: Record<string, any>;
     } = {},
   ): Promise<any> {
     this._ensureAuthenticated();
@@ -475,8 +475,8 @@ export class r2rClient {
         ? JSON.stringify(options.document_ids)
         : undefined,
       user_ids: options.user_ids ? JSON.stringify(options.user_ids) : undefined,
-      chunking_settings: options.chunking_settings
-        ? JSON.stringify(options.chunking_settings)
+      chunking_config: options.chunking_config
+        ? JSON.stringify(options.chunking_config)
         : undefined,
     };
 
@@ -514,7 +514,7 @@ export class r2rClient {
     options: {
       document_ids: string[];
       metadatas?: Record<string, any>[];
-      chunking_settings?: Record<string, any>;
+      chunking_config?: Record<string, any>;
     },
   ): Promise<any> {
     this._ensureAuthenticated();
@@ -552,8 +552,8 @@ export class r2rClient {
       metadatas: options.metadatas
         ? JSON.stringify(options.metadatas)
         : undefined,
-      chunking_settings: options.chunking_settings
-        ? JSON.stringify(options.chunking_settings)
+      chunking_config: options.chunking_config
+        ? JSON.stringify(options.chunking_config)
         : undefined,
     };
 

py/cli/commands/server.py

@@ -180,6 +180,11 @@ def generate_report():
     default=False,
     help="Exclude Postgres from Docker setup",
 )
+@click.option(
+    "--exclude-hatchet",
+    default=False,
+    help="Exclude Hatchet from Docker setup",
+)
 @click.option("--project-name", default="r2r", help="Project name for Docker")
 @click.option("--image", help="Docker image to use")
 @click.option(
@@ -199,6 +204,7 @@ def serve(
     exclude_neo4j,
     exclude_ollama,
     exclude_postgres,
+    exclude_hatchet,
     project_name,
     image,
     config_name,
@@ -224,6 +230,7 @@ def serve(
             exclude_neo4j,
             exclude_ollama,
             exclude_postgres,
+            exclude_hatchet,
             project_name,
             image,
             config_name,

py/cli/utils/docker_utils.py

@@ -14,7 +14,7 @@
 
 def bring_down_docker_compose(project_name, volumes, remove_orphans):
     compose_files = get_compose_files()
-    docker_command = f"docker compose -f {compose_files['base']} -f {compose_files['neo4j']} -f {compose_files['ollama']} -f {compose_files['postgres']}"
+    docker_command = f"docker compose -f {compose_files['base']} -f {compose_files['neo4j']} -f {compose_files['ollama']} -f {compose_files['postgres']} -f {compose_files['hatchet']}"
     docker_command += f" --project-name {project_name}"
 
     if volumes:
@@ -111,6 +111,7 @@ def run_docker_serve(
     exclude_neo4j: bool,
     exclude_ollama: bool,
     exclude_postgres: bool,
+    exclude_hatchet: bool,
     project_name: str,
     image: str,
     config_name: Optional[str] = None,
@@ -137,6 +138,7 @@ def run_docker_serve(
         exclude_neo4j,
         exclude_ollama,
         exclude_postgres,
+        exclude_hatchet,
         project_name,
         config_path,
         image,
@@ -275,6 +277,7 @@ def set_config_env_vars(obj):
     else:
         os.environ["CONFIG_NAME"] = obj.get("config_name") or "default"
 
+
 def get_compose_files():
     package_dir = os.path.join(
         os.path.dirname(os.path.abspath(__file__)),
@@ -286,6 +289,7 @@ def get_compose_files():
         "neo4j": os.path.join(package_dir, "compose.neo4j.yaml"),
         "ollama": os.path.join(package_dir, "compose.ollama.yaml"),
         "postgres": os.path.join(package_dir, "compose.postgres.yaml"),
+        "hatchet": os.path.join(package_dir, "compose.hatchet.yaml"),
     }
 
     for name, path in compose_files.items():
@@ -320,6 +324,7 @@ def build_docker_command(
     exclude_neo4j,
     exclude_ollama,
     exclude_postgres,
+    exclude_hatchet,
     project_name,
     config_path,
     image,
@@ -333,6 +338,8 @@ def build_docker_command(
         command += f" -f {compose_files['ollama']}"
     if not exclude_postgres:
         command += f" -f {compose_files['postgres']}"
+    if not exclude_hatchet:
+        command += f" -f {compose_files['hatchet']}"
 
     command += f" --project-name {project_name}"
 

py/compose.hatchet.yaml

@@ -0,0 +1,150 @@
+version: "3.8"
+
+networks:
+  r2r-network:
+    name: r2r-network
+
+services:
+  hatchet-postgres:
+    image: postgres:15.6
+    command: postgres -c 'max_connections=200'
+    restart: always
+    hostname: "hatchet-postgres"
+    environment:
+      - POSTGRES_USER=hatchet
+      - POSTGRES_PASSWORD=hatchet
+      - POSTGRES_DB=hatchet
+    ports:
+      - "5435:5432"
+    volumes:
+      - hatchet_postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-d", "hatchet"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 10s
+    networks:
+      - r2r-network
+
+  hatchet-rabbitmq:
+    image: "rabbitmq:3-management"
+    hostname: "hatchet-rabbitmq"
+    ports:
+      - "5673:5672"
+      - "15673:15672"
+    environment:
+      RABBITMQ_DEFAULT_USER: "user"
+      RABBITMQ_DEFAULT_PASS: "password"
+    volumes:
+      - "hatchet_rabbitmq_data:/var/lib/rabbitmq"
+      - "hatchet_rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
+    healthcheck:
+      test: ["CMD", "rabbitmqctl", "status"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+    networks:
+      - r2r-network
+
+  hatchet-migration:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-migrate:latest
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@hatchet-postgres:5432/hatchet"
+    depends_on:
+      hatchet-postgres:
+        condition: service_healthy
+    networks:
+      - r2r-network
+
+  hatchet-setup-config:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-admin:latest
+    command: /hatchet/hatchet-admin quickstart --skip certs --generated-config-dir /hatchet/config --overwrite=false
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@hatchet-postgres:5432/hatchet"
+      DATABASE_POSTGRES_PORT: "5432"
+      DATABASE_POSTGRES_HOST: "hatchet-postgres"
+      SERVER_TASKQUEUE_RABBITMQ_URL: amqp://user:password@hatchet-rabbitmq:5672/
+      SERVER_AUTH_COOKIE_DOMAIN: localhost:8080
+      SERVER_AUTH_COOKIE_INSECURE: "t"
+      SERVER_GRPC_BIND_ADDRESS: "0.0.0.0"
+      SERVER_GRPC_INSECURE: "t"
+      SERVER_GRPC_BROADCAST_ADDRESS: localhost:7077
+      SERVER_GRPC_MAX_MSG_SIZE: 104857600
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+    depends_on:
+      hatchet-migration:
+        condition: service_completed_successfully
+      hatchet-rabbitmq:
+        condition: service_healthy
+      hatchet-postgres:
+        condition: service_healthy
+    networks:
+      - r2r-network
+
+  hatchet-engine:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-engine:latest
+    command: /hatchet/hatchet-engine --config /hatchet/config
+    restart: on-failure
+    depends_on:
+      hatchet-setup-config:
+        condition: service_completed_successfully
+      hatchet-migration:
+        condition: service_completed_successfully
+    ports:
+      - "7077:7070"
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@hatchet-postgres:5432/hatchet"
+      SERVER_GRPC_BIND_ADDRESS: "0.0.0.0"
+      SERVER_GRPC_INSECURE: "t"
+      SERVER_GRPC_MAX_MSG_SIZE: 104857600
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+    networks:
+      - r2r-network
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7070/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  hatchet-api:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-api:latest
+    command: /hatchet/hatchet-api --config /hatchet/config
+    restart: on-failure
+    depends_on:
+      hatchet-setup-config:
+        condition: service_completed_successfully
+      hatchet-migration:
+        condition: service_completed_successfully
+    environment:
+      DATABASE_URL: "postgres://hatchet:hatchet@hatchet-postgres:5432/hatchet"
+    volumes:
+      - hatchet_certs:/hatchet/certs
+      - hatchet_config:/hatchet/config
+    networks:
+      - r2r-network
+
+  hatchet-frontend:
+    image: ghcr.io/hatchet-dev/hatchet/hatchet-frontend:latest
+    networks:
+      - r2r-network
+
+  hatchet-caddy:
+    image: caddy:2.7.6-alpine
+    ports:
+      - 8081:8080
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile
+    networks:
+      - r2r-network
+
+volumes:
+  hatchet_postgres_data:
+  hatchet_rabbitmq_data:
+  hatchet_rabbitmq.conf:
+  hatchet_config:
+  hatchet_certs:

py/compose.yaml

@@ -5,6 +5,8 @@ x-depends-on:
     condition: service_healthy
   postgres: &postgres-dependency
     condition: service_healthy
+  hatchet-engine: &hatchet-dependency
+    condition: service_healthy
 
 networks:
   r2r-network:

py/core/base/abstractions/base.py

@@ -1,4 +1,31 @@
 import asyncio
+from typing import Any, Type, TypeVar
+
+from pydantic import BaseModel
+
+T = TypeVar("T", bound="R2RSerializable")
+
+
+class R2RSerializable(BaseModel):
+    @classmethod
+    def from_dict(cls: Type[T], data: dict[str, Any]) -> T:
+        return cls(**data)
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.dict(exclude_unset=True)
+
+    def to_json(self) -> str:
+        return self.json(exclude_unset=True)
+
+    @classmethod
+    def from_json(cls: Type[T], json_str: str) -> T:
+        return cls.parse_raw(json_str)
+
+    class Config:
+        arbitrary_types_allowed = True
+        json_encoders = {
+            bytes: lambda v: v.decode("utf-8", errors="ignore"),
+        }
 
 
 class AsyncSyncMeta(type):