From b112da8bd04dc2fca5b29734fe15f6ba2d965081 Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Tue, 22 Jul 2025 00:51:09 +0200 Subject: [PATCH] Supervisor based run --- app_urls/Dockerfile | 8 ++++++-- app_urls/fetcher/views_base.py | 2 +- app_urls/run.sh | 18 ------------------ app_urls/supervisord.conf | 34 ++++++++++++++++++++++++++++++++++ docker-compose-base.yml | 10 ++++++++++ docker-compose-dev.yml | 7 +++++++ docker-compose.yml | 9 ++++++++- 7 files changed, 66 insertions(+), 22 deletions(-) delete mode 100755 app_urls/run.sh create mode 100644 app_urls/supervisord.conf diff --git a/app_urls/Dockerfile b/app_urls/Dockerfile index 15b92b9..b8d6713 100644 --- a/app_urls/Dockerfile +++ b/app_urls/Dockerfile @@ -5,6 +5,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 #Prevents Python from buffering stdout and stderr ENV PYTHONUNBUFFERED=1 +# supervisor +RUN apt-get update && apt-get install -y supervisor + # User RUN useradd -m -r appuser && \ mkdir /opt/app && \ @@ -14,10 +17,11 @@ WORKDIR /opt/app # Copy the Django project and install dependencies COPY requirements.txt /opt/app/ -# run this command to install all dependencies +# Install dependencies RUN pip install --no-cache-dir -r requirements.txt COPY --chown=appuser:appuser . /opt/app/ +COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf RUN chmod -R 755 /opt RUN chown -R appuser:appuser /opt @@ -25,4 +29,4 @@ RUN chown -R appuser:appuser /opt USER appuser # Run Django’s server & workers -CMD ["sh", "-c", "/opt/app/initialize.sh && /opt/app/run.sh"] +CMD ["sh", "-c", "/opt/app/initialize.sh && /usr/bin/supervisord"] diff --git a/app_urls/fetcher/views_base.py b/app_urls/fetcher/views_base.py index 68e0ad8..175ce0c 100644 --- a/app_urls/fetcher/views_base.py +++ b/app_urls/fetcher/views_base.py @@ -21,7 +21,7 @@ def link_list(request): # List of links list_links = \ [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \ - [ os.path.join(app_url, "logs", log_type) for log_type in ["database", "debug", "info", "warning"] ] #+ \ + [ os.path.join(app_url, "logs", log_type) for log_type in ["database", "debug", "info", "warning", "server", "beat", "worker_default", "worker_low"] ] #+ \ #[ os.path.join(app_url, "task", l) for l in links_fetch + links_process ] # Links tuple diff --git a/app_urls/run.sh b/app_urls/run.sh deleted file mode 100755 index 6d4ab06..0000000 --- a/app_urls/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ "${DJANGO_DEBUG}" = true ] | [ "${DJANGO_DEBUG}" == "True" ]; then - echo "Running in DEBUG mode" - gunicorn core.wsgi:application --reload --log-level debug --bind 0.0.0.0:8000 --timeout 600 & python manage.py scheduler_worker high default low -else - echo "Running in PROD mode" - # Multi-worker - # gunicorn core.wsgi:application --bind 0.0.0.0:8000 --timeout 86400 & while true; do echo "Initializing worker default" >> /opt/logs/warning.log; python manage.py scheduler_worker -v 2 --traceback default high 2>> /opt/logs/warning.log; done & while true; do echo "Initializing worker low" >> /opt/logs/warning.log; python manage.py scheduler_worker -v 2 --without-scheduler --traceback low 2>> /opt/logs/warning.log; done - # - #(sleep 10; while true; do echo "Initializing worker default" >> /opt/logs/info.log; python manage.py scheduler_worker -v 1 --worker-ttl 172800 --traceback --name default default high; sleep 120; done) & - #(sleep 10; while true; do echo "Initializing worker low" >> /opt/logs/info.log; python manage.py scheduler_worker -v 1 --worker-ttl 172800 --traceback --name low low; sleep 120; done) & - # - celery -A core beat -l info & - celery -A core worker -l info --concurrency=1 -Q default & - celery -A core worker -l info --concurrency=1 -Q low & - gunicorn core.wsgi:application --bind 0.0.0.0:8000 --timeout 172800 -fi diff --git a/app_urls/supervisord.conf b/app_urls/supervisord.conf new file mode 100644 index 0000000..3977785 --- /dev/null +++ b/app_urls/supervisord.conf @@ -0,0 +1,34 @@ +[supervisord] +nodaemon=true + +[program:server] +command=gunicorn core.wsgi:application --bind 0.0.0.0:8000 +directory=/opt/app +autostart=true +autorestart=true +stdout_logfile=/opt/logs/server.log +stderr_logfile=/opt/logs/server.log + +[program:beat] +command=celery -A core beat -l info --logfile=/opt/logs/beat.log +directory=/opt/app +autostart=true +autorestart=true +stdout_logfile=/opt/logs/beat.log +stderr_logfile=/opt/logs/beat.log + +[program:worker_default] +command=celery -A core worker -l info --logfile=/opt/logs/worker_default.log --concurrency=1 -Q default -n default +directory=/opt/app +autostart=true +autorestart=true +stdout_logfile=/opt/logs/worker_default.log +stderr_logfile=/opt/logs/worker_default.log + +[program:worker_low] +command=celery -A core worker -l info --logfile=/opt/logs/worker_low.log --concurrency=1 -Q low -n low +directory=/opt/app +autostart=true +autorestart=true +stdout_logfile=/opt/logs/worker_low.log +stderr_logfile=/opt/logs/worker_low.log diff --git a/docker-compose-base.yml b/docker-compose-base.yml index 14c85e5..81c3c21 100644 --- a/docker-compose-base.yml +++ b/docker-compose-base.yml @@ -84,3 +84,13 @@ services: fetcher_db: container_name: fetcher_db restart: unless-stopped + + fetcher_flower: + image: mher/flower + container_name: fetcher_flower + ports: + - 5555 + environment: + - CELERY_BROKER_URL=redis://fetcher_redis:6379/0 + depends_on: + - fetcher_redis diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 563c784..3ad4f6a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -58,3 +58,10 @@ services: service: fetcher_redis ports: - 6379:6379 + + fetcher_flower: + extends: + file: docker-compose-base.yml + service: fetcher_flower + ports: + - 5555:5555 diff --git a/docker-compose.yml b/docker-compose.yml index 1992b1a..51cc79f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,4 +52,11 @@ services: file: docker-compose-base.yml service: fetcher_redis ports: - - 6379:6379 \ No newline at end of file + - 6379:6379 + + fetcher_flower: + extends: + file: docker-compose-base.yml + service: fetcher_flower + ports: + - 5555:5555