Redis cache and celery, avoid overflow

.env.sample

@@ -24,8 +24,10 @@ DB_PASSWORD=supermatitos
 DB_USER=supermatitos
 DB_HOST=fetcher_db
 DB_PORT=5432
-REDIS_HOST=fetcher_redis
+REDIS_CACHE_HOST=fetcher_redis_cache
-REDIS_PORT=6379
+REDIS_CACHE_PORT=6379
+REDIS_CELERY_HOST=fetcher_redis_celery
+REDIS_CELERY_PORT=6380
 
 # Job timeout: 30 min
 JOB_DEFAULT_TIMEOUT=1800

app_urls/core/settings.py

@@ -97,9 +97,10 @@ DATABASES = {
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://{}:{}".format(
+        "LOCATION": "redis://{}:{}/{}".format(
-            os.environ.get("REDIS_HOST", "localhost"), 
+            os.environ.get("REDIS_CACHE_HOST", "localhost"), 
-            os.environ.get("REDIS_PORT", 6379)
+            os.environ.get("REDIS_CACHE_PORT", 6379),
+            2 # DB for Caching
         ),
         "OPTIONS": {
             "MEMCACHE_MAX_KEY_LENGTH": 2048,
@@ -108,13 +109,12 @@ CACHES = {
     }
 }
 
-
-
 # Celery configuration
-CELERY_BROKER_URL = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0))
+CELERY_BROKER_URL = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_CELERY_HOST", "localhost"), os.environ.get("REDIS_CELERY_PORT", 6379), 0)
-CELERY_RESULT_BACKEND = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB_RESULTS", 1))
+CELERY_RESULT_BACKEND = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_CELERY_HOST", "localhost"), os.environ.get("REDIS_CELERY_PORT", 6379), 1)
 CELERY_ACCEPT_CONTENT = ['json']
 CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_EXPIRES = 3600  # Auto clean results after 1 hour
 
 # Celery Beat scheduler (required for django-celery-beat to work)
 CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler'

docker-compose-base.yml

@@ -43,8 +43,10 @@ services:
       - DB_PASSWORD=${DB_PASSWORD}
       - DB_HOST=${DB_HOST}
       - DB_PORT=${DB_PORT}
-      - REDIS_HOST=${REDIS_HOST}
+      - REDIS_CACHE_HOST=${REDIS_CACHE_HOST}
-      - REDIS_PORT=${REDIS_PORT}
+      - REDIS_CACHE_PORT=${REDIS_CACHE_PORT}
+      - REDIS_CELERY_HOST=${REDIS_CELERY_HOST}
+      - REDIS_CELERY_PORT=${REDIS_CELERY_PORT}
       # Job timeout: 30 min
       - JOB_DEFAULT_TIMEOUT=${JOB_DEFAULT_TIMEOUT}
       # Fetcher
@@ -71,18 +73,26 @@ services:
       - 8000
     depends_on:
       - fetcher_db
-      - fetcher_redis
+      - fetcher_redis_cache
+      - fetcher_redis_celery
       - fetcher_app_selenium
     dns:
       - 1.1.1.1
       - 1.0.0.1
 
-  fetcher_redis:
+  fetcher_redis_cache:
     image: redis:alpine
-    container_name: fetcher_redis
+    container_name: fetcher_redis_cache
     restart: unless-stopped
     ports:
       - 6379
+  
+  fetcher_redis_celery:
+    image: redis:alpine
+    container_name: fetcher_redis_celery
+    restart: unless-stopped
+    ports:
+      - 6380
 
   fetcher_db:
     container_name: fetcher_db
@@ -94,6 +104,6 @@ services:
     ports:
       - 5555
     environment:
-      - CELERY_BROKER_URL=redis://fetcher_redis:6379/0
+      - CELERY_BROKER_URL=redis://fetcher_redis_celery:6380/0
     depends_on:
-      - fetcher_redis
+      - fetcher_redis_celery

docker-compose-dev.yml

@@ -52,13 +52,20 @@ services:
     #volumes:   # Persistent DB?
     #  - ./postgres:/var/lib/postgresql/data
 
-  fetcher_redis:
+  fetcher_redis_cache:
     extends:
       file: docker-compose-base.yml
-      service: fetcher_redis
+      service: fetcher_redis_cache
     ports:
       - 6379:6379
 
+  fetcher_redis_celery:
+    extends:
+      file: docker-compose-base.yml
+      service: fetcher_redis_celery
+    ports:
+      - 6380:6380
+
   fetcher_flower:
     extends:
       file: docker-compose-base.yml

docker-compose.yml

@@ -47,13 +47,20 @@ services:
         autossh -M 15885 -N -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST}
         # autossh -M 15885 -N -o 'GatewayPorts yes' -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST}
 
-  fetcher_redis:
+  fetcher_redis_cache:
     extends:
       file: docker-compose-base.yml
-      service: fetcher_redis
+      service: fetcher_redis_cache
     ports:
       - 6379:6379
 
+  fetcher_redis_celery:
+    extends:
+      file: docker-compose-base.yml
+      service: fetcher_redis_celery
+    ports:
+      - 6380:6380
+
   fetcher_flower:
     extends:
       file: docker-compose-base.yml