Supervisor based run

app_urls/Dockerfile

@@ -5,6 +5,9 @@ ENV PYTHONDONTWRITEBYTECODE=1
 #Prevents Python from buffering stdout and stderr
 ENV PYTHONUNBUFFERED=1
 
+# supervisor
+RUN apt-get update && apt-get install -y supervisor
+
 # User
 RUN useradd -m -r appuser && \
    mkdir /opt/app && \
@@ -14,10 +17,11 @@ WORKDIR /opt/app
 
 # Copy the Django project and install dependencies
 COPY requirements.txt  /opt/app/
-# run this command to install all dependencies 
+# Install dependencies 
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY --chown=appuser:appuser . /opt/app/
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
 RUN chmod -R 755 /opt
 RUN chown -R appuser:appuser /opt
@@ -25,4 +29,4 @@ RUN chown -R appuser:appuser /opt
 USER appuser
 
 # Run Django’s server & workers
-CMD ["sh", "-c", "/opt/app/initialize.sh && /opt/app/run.sh"]
+CMD ["sh", "-c", "/opt/app/initialize.sh && /usr/bin/supervisord"]

app_urls/fetcher/views_base.py

@@ -21,7 +21,7 @@ def link_list(request):
     # List of links
     list_links = \
         [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \
-        [ os.path.join(app_url, "logs", log_type) for log_type in ["database", "debug", "info", "warning"] ] #+ \
+        [ os.path.join(app_url, "logs", log_type) for log_type in ["database", "debug", "info", "warning", "server", "beat", "worker_default", "worker_low"] ] #+ \
         #[ os.path.join(app_url, "task", l) for l in links_fetch + links_process ]
 
     # Links tuple

app_urls/run.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-if [ "${DJANGO_DEBUG}" = true ] | [ "${DJANGO_DEBUG}" == "True" ]; then
-    echo "Running in DEBUG mode"
-    gunicorn core.wsgi:application --reload --log-level debug --bind 0.0.0.0:8000 --timeout 600 & python manage.py scheduler_worker high default low
-else
-    echo "Running in PROD mode"
-    # Multi-worker
-    # gunicorn core.wsgi:application --bind 0.0.0.0:8000 --timeout 86400 & while true; do echo "Initializing worker default" >> /opt/logs/warning.log; python manage.py scheduler_worker -v 2 --traceback default high 2>> /opt/logs/warning.log; done & while true; do echo "Initializing worker low" >> /opt/logs/warning.log; python manage.py scheduler_worker -v 2 --without-scheduler --traceback low 2>> /opt/logs/warning.log; done
-    #
-    #(sleep 10; while true; do echo "Initializing worker default" >> /opt/logs/info.log; python manage.py scheduler_worker -v 1 --worker-ttl 172800 --traceback --name default default high; sleep 120; done) &
-    #(sleep 10; while true; do echo "Initializing worker low" >> /opt/logs/info.log; python manage.py scheduler_worker -v 1 --worker-ttl 172800 --traceback --name low low; sleep 120; done) &
-    #
-    celery -A core beat -l info &
-    celery -A core worker -l info --concurrency=1 -Q default &
-    celery -A core worker -l info --concurrency=1 -Q low &
-    gunicorn core.wsgi:application --bind 0.0.0.0:8000 --timeout 172800
-fi

app_urls/supervisord.conf

@@ -0,0 +1,34 @@
+[supervisord]
+nodaemon=true
+
+[program:server]
+command=gunicorn core.wsgi:application --bind 0.0.0.0:8000
+directory=/opt/app
+autostart=true
+autorestart=true
+stdout_logfile=/opt/logs/server.log
+stderr_logfile=/opt/logs/server.log
+
+[program:beat]
+command=celery -A core beat -l info --logfile=/opt/logs/beat.log
+directory=/opt/app
+autostart=true
+autorestart=true
+stdout_logfile=/opt/logs/beat.log
+stderr_logfile=/opt/logs/beat.log
+
+[program:worker_default]
+command=celery -A core worker -l info --logfile=/opt/logs/worker_default.log --concurrency=1 -Q default -n default
+directory=/opt/app
+autostart=true
+autorestart=true
+stdout_logfile=/opt/logs/worker_default.log
+stderr_logfile=/opt/logs/worker_default.log
+
+[program:worker_low]
+command=celery -A core worker -l info --logfile=/opt/logs/worker_low.log --concurrency=1 -Q low -n low
+directory=/opt/app
+autostart=true
+autorestart=true
+stdout_logfile=/opt/logs/worker_low.log
+stderr_logfile=/opt/logs/worker_low.log

docker-compose-base.yml

@@ -84,3 +84,13 @@ services:
   fetcher_db:
     container_name: fetcher_db
     restart: unless-stopped
+
+  fetcher_flower:
+    image: mher/flower
+    container_name: fetcher_flower
+    ports:
+      - 5555
+    environment:
+      - CELERY_BROKER_URL=redis://fetcher_redis:6379/0
+    depends_on:
+      - fetcher_redis

docker-compose-dev.yml

@@ -58,3 +58,10 @@ services:
       service: fetcher_redis
     ports:
       - 6379:6379
+
+  fetcher_flower:
+    extends:
+      file: docker-compose-base.yml
+      service: fetcher_flower
+    ports:
+      - 5555:5555

docker-compose.yml

@@ -52,4 +52,11 @@ services:
       file: docker-compose-base.yml
       service: fetcher_redis
     ports:
-      - 6379:6379
+      - 6379:6379
+
+  fetcher_flower:
+    extends:
+      file: docker-compose-base.yml
+      service: fetcher_flower
+    ports:
+      - 5555:5555