Merge pull request #1871 from Agenta-AI/migration-and-cleanup

docker compose cleanup from mongo post-migration

agenta-backend/agenta_backend/migrations/mongo_to_postgres/README.md

@@ -1,6 +1,2 @@
-```bash
-docker ps 
-docker exec -it {backend-container-id} bash
-cd /app/agenta_backend/migrations/mongo_to_postgres
-python3 migration.py
-```
+Migration steps are detailed here:
+https://docs.agenta.ai/self-host/migration/migration-to-postgres

agenta-backend/agenta_backend/migrations/mongo_to_postgres/docker-compose.migration.yml

@@ -0,0 +1,220 @@
+services:
+    reverse-proxy:
+        image: traefik:v2.10
+        command: --api.dashboard=true --api.insecure=true --providers.docker --entrypoints.web.address=:80
+        ports:
+            - "80:80"
+            - "8080:8080"
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+        networks:
+            - agenta-network
+        restart: always
+
+    backend:
+        build: ./agenta-backend
+        environment:
+            - MONGODB_URI=mongodb://username:password@mongo:27017
+            - POSTGRES_URI=postgresql+asyncpg://username:password@postgres:5432/agenta_oss
+            - REDIS_URL=redis://redis:6379/0
+            - ENVIRONMENT=development
+            - DATABASE_MODE=v2
+            - MIGRATION_SRC_MONGO_DB_NAME=v2
+            - BARE_DOMAIN_NAME=localhost
+            - DOMAIN_NAME=http://localhost
+            - FEATURE_FLAG=oss
+            - AGENTA_TEMPLATE_REPO=agentaai/templates_v2
+            - POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7
+            - CELERY_BROKER_URL=amqp://guest@rabbitmq//
+            - CELERY_RESULT_BACKEND=redis://redis:6379/0
+            - TEMPLATES_BASE_URL=https://llm-app-json.s3.eu-central-1.amazonaws.com
+            - REGISTRY_REPO_NAME=agentaai
+            - DOCKER_HUB_URL=https://hub.docker.com/v2/repositories
+        volumes:
+            - ./agenta-backend/agenta_backend:/app/agenta_backend
+            - ./agenta-backend/tests:/app/tests
+            - /var/run/docker.sock:/var/run/docker.sock
+            - ./agenta-backend/db:/db
+        labels:
+            - "traefik.http.routers.backend.rule=PathPrefix(`/api/`)"
+            - "traefik.http.routers.backend.entrypoints=web"
+            - "traefik.http.middlewares.backend-strip.stripprefix.prefixes=/api"
+            - "traefik.http.middlewares.backend-strip.stripprefix.forceslash=true"
+            - "traefik.http.routers.backend.middlewares=backend-strip"
+            - "traefik.http.services.backend.loadbalancer.server.port=8000"
+            - "traefik.http.routers.backend.service=backend"
+        networks:
+            - agenta-network
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        command:
+            [
+                "uvicorn",
+                "agenta_backend.main:app",
+                "--host",
+                "0.0.0.0",
+                "--port",
+                "8000",
+                "--reload",
+                "--log-level",
+                "info",
+                "--root-path",
+                "/api",
+            ]
+        depends_on:
+            mongo:
+                condition: service_healthy
+            postgres:
+                condition: service_healthy
+        restart: always
+
+    agenta-web:
+        build:
+            context: ./agenta-web
+            dockerfile: dev.Dockerfile
+        volumes:
+            - ./agenta-web/src:/app/src
+            - ./agenta-web/public:/app/public
+            - .nextjs_cache:/app/.next
+        ports:
+            - "3000:3000"
+        networks:
+            - agenta-network
+        labels:
+            - "traefik.http.routers.agenta-web.rule=PathPrefix(`/`)"
+            - "traefik.http.routers.agenta-web.entrypoints=web"
+            - "traefik.http.services.agenta-web.loadbalancer.server.port=3000"
+        environment:
+            - NEXT_PUBLIC_POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7
+        restart: always
+
+    mongo:
+        image: mongo:5.0
+        environment:
+            MONGO_INITDB_ROOT_USERNAME: username
+            MONGO_INITDB_ROOT_PASSWORD: password
+        volumes:
+            - mongodb_data:/data/db
+        ports:
+            - "27017:27017"
+        networks:
+            - agenta-network
+        healthcheck:
+            test: ["CMD", "mongo", "--eval", "db.adminCommand('ping')"]
+            interval: 10s
+            timeout: 10s
+            retries: 20
+        restart: always
+
+    mongo_express:
+        image: mongo-express:0.54.0
+        environment:
+            ME_CONFIG_MONGODB_ADMINUSERNAME: username
+            ME_CONFIG_MONGODB_ADMINPASSWORD: password
+            ME_CONFIG_MONGODB_SERVER: mongo
+        ports:
+            - "8081:8081"
+        networks:
+            - agenta-network
+        depends_on:
+            mongo:
+                condition: service_healthy
+        restart: always
+
+    redis:
+        image: redis:latest
+        networks:
+            - agenta-network
+        volumes:
+            - redis_data:/data
+        restart: always
+
+    rabbitmq:
+        image: rabbitmq:3-management
+        ports:
+            - "5672:5672"
+            - "15672:15672"
+        volumes:
+            - ./rabbitmq_data:/var/lib/rabbitmq
+        environment:
+            RABBITMQ_DEFAULT_USER: "guest"
+            RABBITMQ_DEFAULT_PASS: "guest"
+        networks:
+            - agenta-network
+
+    celery_worker:
+        build: ./agenta-backend
+        command: >
+            watchmedo auto-restart --directory=./agenta_backend --pattern=*.py --recursive -- celery -A agenta_backend.main.celery_app worker --concurrency=1 --loglevel=INFO
+        environment:
+            - POSTGRES_URI=postgresql+asyncpg://username:password@postgres:5432/agenta_oss
+            - MONGODB_URI=mongodb://username:password@mongo:27017
+            - REDIS_URL=redis://redis:6379/0
+            - CELERY_BROKER_URL=amqp://guest@rabbitmq//
+            - CELERY_RESULT_BACKEND=redis://redis:6379/0
+            - FEATURE_FLAG=oss
+        volumes:
+            - ./agenta-backend/agenta_backend:/app/agenta_backend
+            - /var/run/docker.sock:/var/run/docker.sock
+        depends_on:
+            - mongo
+            - postgres
+            - rabbitmq
+            - redis
+        extra_hosts:
+            - "host.docker.internal:host-gateway"
+        networks:
+            - agenta-network
+
+    postgres:
+        image: postgres:16.2
+        container_name: postgres
+        restart: always
+        environment:
+            POSTGRES_USER: username
+            POSTGRES_PASSWORD: password
+            POSTGRES_DB: agenta_oss
+        ports:
+            - "5432:5432"
+        networks:
+            - agenta-network
+        volumes:
+            - postgresdb-data:/var/lib/postgresql/data/
+            - ./docker-assets/postgres/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
+        healthcheck:
+            test: ["CMD-SHELL", "pg_isready -U postgres"]
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+    pgadmin:
+        image: dpage/pgadmin4
+        restart: always
+        environment:
+            PGADMIN_DEFAULT_EMAIL: "[email protected]"
+            PGADMIN_DEFAULT_PASSWORD: "password"
+            PGADMIN_SERVER_HOST: "postgres"
+            PGADMIN_SERVER_PORT: 5432
+            PGADMIN_SERVER_USER: "username"
+            PGADMIN_SERVER_PASSWORD: "password"
+            PGADMIN_SERVER_DB: agenta_oss
+        ports:
+            - "5050:80"
+        networks:
+            - agenta-network
+        volumes:
+            - pgadmin-data:/var/lib/pgadmin
+        depends_on:
+            postgres:
+                condition: service_healthy
+
+networks:
+    agenta-network:
+        name: agenta-network
+
+volumes:
+    mongodb_data:
+    redis_data:
+    nextjs_cache:
+    postgresdb-data:
+    pgadmin-data:

docker-compose.gh.yml

@@ -15,12 +15,10 @@ services:
         container_name: agenta-backend-1
         image: ghcr.io/agenta-ai/agenta-backend
         environment:
-            - MONGODB_URI=mongodb://username:password@mongo:27017
             - POSTGRES_URI=postgresql+asyncpg://username:password@postgres:5432/agenta_oss
             - REDIS_URL=redis://redis:6379/0
             - ENVIRONMENT=production
             - DATABASE_MODE=v2
-            - MIGRATION_SRC_MONGO_DB_NAME=v2
             - FEATURE_FLAG=oss
             - AGENTA_TEMPLATE_REPO=agentaai/templates_v2
             - POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7
@@ -43,7 +41,7 @@ services:
                 "--log-level",
                 "info",
                 "--root-path",
-                "/api"
+                "/api",
             ]
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
@@ -60,8 +58,6 @@ services:
         extra_hosts:
             - "host.docker.internal:host-gateway"
         depends_on:
-            mongo:
-                condition: service_healthy
             postgres:
                 condition: service_healthy
         restart: always
@@ -84,39 +80,6 @@ services:
             - NEXT_PUBLIC_TELEMETRY_TRACKING_ENABLED=true
         restart: always
 
-    mongo:
-        image: mongo:5.0
-        environment:
-            MONGO_INITDB_ROOT_USERNAME: username
-            MONGO_INITDB_ROOT_PASSWORD: password
-        volumes:
-            - mongodb_data:/data/db
-        ports:
-            - "27017:27017"
-        networks:
-            - agenta-network
-        healthcheck:
-            test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
-            interval: 10s
-            timeout: 10s
-            retries: 20
-        restart: always
-
-    mongo_express:
-        image: mongo-express
-        environment:
-            ME_CONFIG_MONGODB_ADMINUSERNAME: username
-            ME_CONFIG_MONGODB_ADMINPASSWORD: password
-            ME_CONFIG_MONGODB_SERVER: mongo
-        ports:
-            - "8081:8081"
-        networks:
-            - agenta-network
-        depends_on:
-            mongo:
-                condition: service_healthy
-        restart: always
-
     redis:
         image: redis:latest
         networks:
@@ -145,15 +108,13 @@ services:
             celery -A agenta_backend.main.celery_app worker --concurrency=1 --loglevel=INFO
         environment:
             - POSTGRES_URI=postgresql+asyncpg://username:password@postgres:5432/agenta_oss
-            - MONGODB_URI=mongodb://username:password@mongo:27017
             - REDIS_URL=redis://redis:6379/0
             - CELERY_BROKER_URL=amqp://guest@rabbitmq//
             - CELERY_RESULT_BACKEND=redis://redis:6379/0
             - FEATURE_FLAG=oss
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
         depends_on:
-            - mongo
             - postgres
             - rabbitmq
             - redis
@@ -209,7 +170,6 @@ networks:
         name: agenta-network
 
 volumes:
-    mongodb_data:
     redis_data:
     postgresdb-data:
     pgadmin-data: