diff --git a/.env.sample b/.env.sample index 235416f..d7ae437 100644 --- a/.env.sample +++ b/.env.sample @@ -24,8 +24,10 @@ DB_PASSWORD=supermatitos DB_USER=supermatitos DB_HOST=fetcher_db DB_PORT=5432 -REDIS_HOST=fetcher_redis -REDIS_PORT=6379 +REDIS_CACHE_HOST=fetcher_redis_cache +REDIS_CACHE_PORT=6379 +REDIS_CELERY_HOST=fetcher_redis_celery +REDIS_CELERY_PORT=6380 # Job timeout: 30 min JOB_DEFAULT_TIMEOUT=1800 diff --git a/app_urls/core/settings.py b/app_urls/core/settings.py index 4317c2e..e46f48d 100644 --- a/app_urls/core/settings.py +++ b/app_urls/core/settings.py @@ -97,9 +97,10 @@ DATABASES = { CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{}:{}".format( - os.environ.get("REDIS_HOST", "localhost"), - os.environ.get("REDIS_PORT", 6379) + "LOCATION": "redis://{}:{}/{}".format( + os.environ.get("REDIS_CACHE_HOST", "localhost"), + os.environ.get("REDIS_CACHE_PORT", 6379), + 2 # DB for Caching ), "OPTIONS": { "MEMCACHE_MAX_KEY_LENGTH": 2048, @@ -108,13 +109,12 @@ CACHES = { } } - - # Celery configuration -CELERY_BROKER_URL = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0)) -CELERY_RESULT_BACKEND = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB_RESULTS", 1)) +CELERY_BROKER_URL = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_CELERY_HOST", "localhost"), os.environ.get("REDIS_CELERY_PORT", 6379), 0) +CELERY_RESULT_BACKEND = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_CELERY_HOST", "localhost"), os.environ.get("REDIS_CELERY_PORT", 6379), 1) CELERY_ACCEPT_CONTENT = ['json'] CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_EXPIRES = 3600 # Auto clean results after 1 hour # Celery Beat scheduler (required for django-celery-beat to work) CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' diff --git a/docker-compose-base.yml b/docker-compose-base.yml index c67a706..4c87732 100644 --- a/docker-compose-base.yml +++ b/docker-compose-base.yml @@ -43,8 +43,10 @@ services: - DB_PASSWORD=${DB_PASSWORD} - DB_HOST=${DB_HOST} - DB_PORT=${DB_PORT} - - REDIS_HOST=${REDIS_HOST} - - REDIS_PORT=${REDIS_PORT} + - REDIS_CACHE_HOST=${REDIS_CACHE_HOST} + - REDIS_CACHE_PORT=${REDIS_CACHE_PORT} + - REDIS_CELERY_HOST=${REDIS_CELERY_HOST} + - REDIS_CELERY_PORT=${REDIS_CELERY_PORT} # Job timeout: 30 min - JOB_DEFAULT_TIMEOUT=${JOB_DEFAULT_TIMEOUT} # Fetcher @@ -71,18 +73,26 @@ services: - 8000 depends_on: - fetcher_db - - fetcher_redis + - fetcher_redis_cache + - fetcher_redis_celery - fetcher_app_selenium dns: - 1.1.1.1 - 1.0.0.1 - fetcher_redis: + fetcher_redis_cache: image: redis:alpine - container_name: fetcher_redis + container_name: fetcher_redis_cache restart: unless-stopped ports: - 6379 + + fetcher_redis_celery: + image: redis:alpine + container_name: fetcher_redis_celery + restart: unless-stopped + ports: + - 6380 fetcher_db: container_name: fetcher_db @@ -94,6 +104,6 @@ services: ports: - 5555 environment: - - CELERY_BROKER_URL=redis://fetcher_redis:6379/0 + - CELERY_BROKER_URL=redis://fetcher_redis_celery:6380/0 depends_on: - - fetcher_redis + - fetcher_redis_celery diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3ad4f6a..ccd7a43 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -52,13 +52,20 @@ services: #volumes: # Persistent DB? # - ./postgres:/var/lib/postgresql/data - fetcher_redis: + fetcher_redis_cache: extends: file: docker-compose-base.yml - service: fetcher_redis + service: fetcher_redis_cache ports: - 6379:6379 + fetcher_redis_celery: + extends: + file: docker-compose-base.yml + service: fetcher_redis_celery + ports: + - 6380:6380 + fetcher_flower: extends: file: docker-compose-base.yml diff --git a/docker-compose.yml b/docker-compose.yml index 51cc79f..d31b2fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,13 +47,20 @@ services: autossh -M 15885 -N -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} # autossh -M 15885 -N -o 'GatewayPorts yes' -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} - fetcher_redis: + fetcher_redis_cache: extends: file: docker-compose-base.yml - service: fetcher_redis + service: fetcher_redis_cache ports: - 6379:6379 + fetcher_redis_celery: + extends: + file: docker-compose-base.yml + service: fetcher_redis_celery + ports: + - 6380:6380 + fetcher_flower: extends: file: docker-compose-base.yml