Merge pull request #283 from lardbit/run-celery-scheduler-separately

Run celery scheduler separately
lardbit · Apr 10, 2024 · cd91ca5 · cd91ca5
2 parents 3037cb2 + ee2c4fd
commit cd91ca5
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 14 deletions.
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
@@ -34,30 +34,49 @@ services:
       - nefarious-db:/nefarious-db
     logging:
       options:
-        max-size: 10m
+        max-size: 2m
 
   # celery base service (to be extended)
   celery-base:
     extends:
       service: nefarious-base
     restart: always
     entrypoint: /app/entrypoint-celery.sh
-    environment:
-      # https://github.com/kennethreitz/dj-database-url
-      DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3"
-      REDIS_HOST: "redis"
-      HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp}
-      INTERNAL_DOWNLOAD_PATH: /downloads
     volumes:
       - ${HOST_DOWNLOAD_PATH:-/tmp}:/downloads
       # persistent named volume for sqlite database
       - nefarious-db:/nefarious-db
     logging:
       options:
-        max-size: 10m
+        max-size: 2m
     labels:
       - com.centurylinklabs.watchtower.enable=true
 
+  # celery worker
+  celery-worker:
+    extends:
+      service: celery-base
+    environment:
+      # https://github.com/kennethreitz/dj-database-url
+      DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3"
+      REDIS_HOST: "redis"
+      HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp}
+      INTERNAL_DOWNLOAD_PATH: /downloads
+      CELERY_BEAT_SEPARATELY: 1  # run worker and scheduler separately
+
+  # celery scheduler (beat)
+  celery-scheduler:
+    extends:
+      service: celery-base
+    environment:
+      # https://github.com/kennethreitz/dj-database-url
+      DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3"
+      REDIS_HOST: "redis"
+      HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp}
+      INTERNAL_DOWNLOAD_PATH: /downloads
+      CELERY_BEAT: 1  # run celery as scheduler
+      CELERY_BEAT_SEPARATELY: 1  # run worker and scheduler separately
+
   # jackett base service (to be extended)
   jackett-base:
     labels:
@@ -67,7 +86,7 @@ services:
     restart: always
     logging:
       options:
-        max-size: 10m
+        max-size: 2m
     volumes:
       - jackett-config:/config
 
@@ -96,7 +115,7 @@ services:
       - WATCHTOWER_POLL_INTERVAL=3600
     logging:
       options:
-        max-size: 10m
+        max-size: 2m
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 

diff --git a/docker-compose.transmission-vpn.yml b/docker-compose.transmission-vpn.yml
@@ -16,7 +16,12 @@ services:
   celery:
     extends:
       file: docker-compose.base.yml
-      service: celery-base
+      service: celery-worker
+
+  celery-scheduler:
+    extends:
+      file: docker-compose.base.yml
+      service: celery-scheduler
 
   redis:
     extends:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,7 +21,15 @@ services:
   # nefarious background task queue
   celery:
     extends:
-      service: celery-base
+      service: celery-worker
+      file: docker-compose.base.yml
+    depends_on:
+      - redis
+
+  # nefarious background task queue scheduler
+  celery-scheduler:
+    extends:
+      service: celery-scheduler
       file: docker-compose.base.yml
     depends_on:
       - redis

diff --git a/entrypoint-celery.sh b/entrypoint-celery.sh
@@ -3,5 +3,19 @@
 # setup
 . $(dirname $0)/entrypoint-base.sh
 
-# run as requested user
-su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --beat --loglevel=INFO"
+# run celery worker/scheduler as requested user
+
+# run celery worker & scheduler separately
+if [ ! -z $CELERY_BEAT_SEPARATELY ]; then
+  # run celery scheduler
+  if [ ! -z $CELERY_BEAT ]; then
+    su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious beat --loglevel=INFO"
+  # run celery workers with defined concurrency
+  else
+    su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --loglevel=INFO"
+  fi
+# run celery worker & scheduler at the same time
+# NOTE: this isn't advised since the scheduler could be potentially running multiple/simultaneous times creating duplicate tasks
+else
+  su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --beat --loglevel=INFO"
+fi