From cb33580ee71e9e5bcc4ed7b59d95aea6897d375b Mon Sep 17 00:00:00 2001 From: lardbit Date: Wed, 10 Apr 2024 11:36:28 -0500 Subject: [PATCH 1/2] run celery scheduler (beat) separately so concurrency doesn't create duplicate tasks, decrease logging sizes --- docker-compose.base.yml | 39 +++++++++++++++++++++-------- docker-compose.transmission-vpn.yml | 7 +++++- docker-compose.yml | 10 +++++++- entrypoint-celery.sh | 17 +++++++++++-- 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 5fa6ea4b..af2b1fde 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -34,7 +34,7 @@ services: - nefarious-db:/nefarious-db logging: options: - max-size: 10m + max-size: 2m # celery base service (to be extended) celery-base: @@ -42,22 +42,41 @@ services: service: nefarious-base restart: always entrypoint: /app/entrypoint-celery.sh - environment: - # https://github.com/kennethreitz/dj-database-url - DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3" - REDIS_HOST: "redis" - HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp} - INTERNAL_DOWNLOAD_PATH: /downloads volumes: - ${HOST_DOWNLOAD_PATH:-/tmp}:/downloads # persistent named volume for sqlite database - nefarious-db:/nefarious-db logging: options: - max-size: 10m + max-size: 2m labels: - com.centurylinklabs.watchtower.enable=true + # celery worker + celery-worker: + extends: + service: celery-base + environment: + # https://github.com/kennethreitz/dj-database-url + DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3" + REDIS_HOST: "redis" + HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp} + INTERNAL_DOWNLOAD_PATH: /downloads + CELERY_BEAT_SEPARATELY: 1 # run worker and scheduler separately + + # celery scheduler (beat) + celery-scheduler: + extends: + service: celery-base + environment: + # https://github.com/kennethreitz/dj-database-url + DATABASE_URL: "sqlite:////nefarious-db/db.sqlite3" + REDIS_HOST: "redis" + HOST_DOWNLOAD_PATH: ${HOST_DOWNLOAD_PATH:-/tmp} + INTERNAL_DOWNLOAD_PATH: /downloads + CELERY_BEAT: 1 # run celery as scheduler + CELERY_BEAT_SEPARATELY: 1 # run worker and scheduler separately + # jackett base service (to be extended) jackett-base: labels: @@ -67,7 +86,7 @@ services: restart: always logging: options: - max-size: 10m + max-size: 2m volumes: - jackett-config:/config @@ -96,7 +115,7 @@ services: - WATCHTOWER_POLL_INTERVAL=3600 logging: options: - max-size: 10m + max-size: 2m volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose.transmission-vpn.yml b/docker-compose.transmission-vpn.yml index 53d26d12..df44014b 100644 --- a/docker-compose.transmission-vpn.yml +++ b/docker-compose.transmission-vpn.yml @@ -16,7 +16,12 @@ services: celery: extends: file: docker-compose.base.yml - service: celery-base + service: celery-worker + + celery-scheduler: + extends: + file: docker-compose.base.yml + service: celery-scheduler redis: extends: diff --git a/docker-compose.yml b/docker-compose.yml index f9169fd4..32cf7965 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,15 @@ services: # nefarious background task queue celery: extends: - service: celery-base + service: celery-worker + file: docker-compose.base.yml + depends_on: + - redis + + # nefarious background task queue scheduler + celery-scheduler: + extends: + service: celery-scheduler file: docker-compose.base.yml depends_on: - redis diff --git a/entrypoint-celery.sh b/entrypoint-celery.sh index f5d7cdbc..766dde39 100755 --- a/entrypoint-celery.sh +++ b/entrypoint-celery.sh @@ -3,5 +3,18 @@ # setup . $(dirname $0)/entrypoint-base.sh -# run as requested user -su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --beat --loglevel=INFO" +# run celery worker/scheduler as requested user + +# run celery worker & scheduler separately +if [ ! -z $CELERY_BEAT_SEPARATELY ]; then + # run celery scheduler + if [ ! -z $CELERY_BEAT ]; then + su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious beat --loglevel=INFO" + # run celery workers with defined concurrency + else + su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --loglevel=INFO" + fi +# run celery worker & scheduler at the same time, with single concurrency +else + su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency 1 --beat --loglevel=INFO" +fi From ee2c4fdd1a23358a4aa29459ba5d8adca7ff9ba6 Mon Sep 17 00:00:00 2001 From: lardbit Date: Wed, 10 Apr 2024 11:56:08 -0500 Subject: [PATCH 2/2] run celery scheduler (beat) separately so concurrency doesn't create duplicate tasks, decrease logging sizes --- entrypoint-celery.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/entrypoint-celery.sh b/entrypoint-celery.sh index 766dde39..5d007246 100755 --- a/entrypoint-celery.sh +++ b/entrypoint-celery.sh @@ -14,7 +14,8 @@ if [ ! -z $CELERY_BEAT_SEPARATELY ]; then else su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --loglevel=INFO" fi -# run celery worker & scheduler at the same time, with single concurrency +# run celery worker & scheduler at the same time +# NOTE: this isn't advised since the scheduler could be potentially running multiple/simultaneous times creating duplicate tasks else - su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency 1 --beat --loglevel=INFO" + su $(id -un ${RUN_AS_UID}) -c "/env/bin/celery -A nefarious worker --concurrency ${NUM_CELERY_WORKERS:-0} --beat --loglevel=INFO" fi