Dockerization, whitenoise serving static, refactor

docker-compose.yml

@@ -2,101 +2,106 @@ version: '3.9'
 
 services:
 
-  fetcher_selenium:
+  fetcher_app_selenium:
+    image: fetcher_app_selenium
     build:
       context: ./app_selenium
-    container_name: selenium_app
-    restart: unless-stopped
+    container_name: fetcher_app_selenium
+    # restart: unless-stopped
     shm_size: 512mb
     environment:
-      - SELENIUM_SLEEP_PER_PAGE=4
-      - PATH_LOGS_PARAMETERIZATION="logs/log_app_selenium_{}.log"
+      - SELENIUM_SLEEP_PER_PAGE=${SELENIUM_SLEEP_PER_PAGE:-4}
+      - PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY:-logs}
     ports:
       - 80
+    dns:
+      - 1.1.1.1
+      - 1.0.0.1
+    deploy:
+      resources:
+        limits:
+          cpus: '4'
+          memory: 4G
 
-  fetcher_urls_app:
+  fetcher_app_urls:
+    image: fetcher_app_urls
     build:
       context: ./app_urls
-    container_name: urls_app
-    restart: unless-stopped
+    container_name: fetcher_app_urls
+    # restart: unless-stopped
     environment:
-      #- name=value
+      # Initialization
+      - INITIALIZE_DB=${INITIALIZE_DB:-true}
+      - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-matitos}
+      - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-matitos}
+      - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-matitos@matitos.org}
+      # Django
+      - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-abc123456789qwerty}
+      - DJANGO_DEBUG=${DJANGO_DEBUG:-False}
+      - DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS:-*} # host1,host2
       # Database
       - DB_NAME=${DB_NAME:-matitos}
-      - DB_USER=${DB_NAME:-supermatitos}
-      - DB_PASSWORD=${DB_NAME:-supermatitos}
-      - DB_HOST=${DB_NAME:-localhost} # db_postgres
-      - DB_PORT=${DB_NAME:-5432}
-      - REDIS_HOST=${REDIS_HOST:-localhost}
+      - DB_USER=${DB_USER:-supermatitos}
+      - DB_PASSWORD=${DB_PASSWORD:-supermatitos}
+      - DB_HOST=${DB_HOST:-fetcher_db}
+      - DB_PORT=${DB_PORT:-5432}
+      - REDIS_HOST=${REDIS_HOST:-fetcher_redis}
       - REDIS_PORT=${REDIS_PORT:-6379}
       # Job timeout: 30 min
       - JOB_DEFAULT_TIMEOUT=${RQ_DEFAULT_TIMEOUT:-1800}
       # Logs path
-      - PATH_LOGS_PARAMETERIZATION="logs/log_app_fetcher_{}.log"
+      - PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY:-logs}
       # Fetcher
-      - FETCHER_GNEWS_DECODE_SLEEP=2
-      - FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP=4
-      - FETCHER_BETWEEN_SEARCHES_SLEEP=5
-      - FETCHER_URL_HOST_SLEEP=5
+      - FETCHER_GNEWS_DECODE_SLEEP=${FETCHER_GNEWS_DECODE_SLEEP-2}
+      - FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP=${FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP:-5}
+      - FETCHER_BETWEEN_SEARCHES_SLEEP=${FETCHER_BETWEEN_SEARCHES_SLEEP:-1}
+      - FETCHER_URL_HOST_SLEEP=${FETCHER_URL_HOST_SLEEP:-2}
       # Selenium
-      - SELENIUM_ENDPOINT="http://selenium_app:80"
+      - SELENIUM_ENDPOINT=${SELENIUM_ENDPOINT:-http://fetcher_app_selenium:80}
+      - ENDPOINT_OLLAMA=${ENDPOINT_OLLAMA:-https://ollamamodel.matitos.org}
     ports:
-      - 80
+      - 8000:8000
+    depends_on:
+      - fetcher_db
+      - fetcher_redis
+    dns:
+      - 1.1.1.1
+      - 1.0.0.1
+    deploy:
+      resources:
+        limits:
+          cpus: '4'
+          memory: 4G
 
   fetcher_db:
     image: postgres:17
-    container_name: db_postgres
+    container_name: fetcher_db
     restart: unless-stopped
     # Set shared memory limit when using docker-compose
     shm_size: 128mb
     environment:
+      POSTGRES_DB: ${DB_NAME:-matitos}
       POSTGRES_PASSWORD: ${DB_PASSWORD:-supermatitos}
-      POSTGRES_USER: ${DB_USERNAME:-supermatitos}
-      POSTGRES_DB: ${DB_DATABASE_NAME:-matitos}
+      POSTGRES_USER: ${DB_USER:-supermatitos}
       POSTGRES_INITDB_ARGS: '--data-checksums'
-    #volumes:
-    #  - ${PATH_BASE:-.}/postgres:/var/lib/postgresql/data
+    #volumes:   # Persistent DB?
+    #  - ${PATH_DB_DATA:-.}/postgres:/var/lib/postgresql/data
     ports:
-      - 5432:5432
+      - 5432 #:5432
 
   fetcher_redis:
     image: redis:alpine
-    container_name: db_redis
+    container_name: fetcher_redis
     restart: unless-stopped
     ports:
-      - 6379:6379
-    #expose:
-    #  - 6379
-
-  fetcher_adminer:
-    # http://localhost:8080/?pgsql=matitos_db&username=supermatitos&db=matitos&ns=public
-    image: adminer
-    container_name: adminer
-    restart: unless-stopped
-    environment:
-      - ADMINER_DEFAULT_DB_DRIVER=pgsql
-      #- ADMINER_DEFAULT_DB_HOST
-      #- ADMINER_DEFAULT_DB_NAME
-    depends_on: 
-      - matitos_db
-    ports:
-      - 8080:8080
+      - 6379 #:6379
 
   fetcher_dozzle:
-    container_name: dozzle
+    container_name: fetcher_dozzle
     image: amir20/dozzle:latest
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
     ports:
       - 8888:8080
     environment:
-      - DOZZLE_FILTER="name=matitos_" # Need container name matitos_ ?
-
-
-# django:
-# Env: DB_HOST=matitos_db
-# DJANGO_DB_NAME=${DB_DATABASE_NAME:-matitos}
-# DJANGO_DB_USER=${DB_USERNAME:-supermatitos}
-# DJANGO_DB_PASSWORD=${DB_PASSWORD:-supermatitos}
-# DJANGO_DB_HOST=${DB_HOST:-localhost}
-# DJANGO_DB_PORT=${DB_PORT:-5432}
+      - DOZZLE_FILTER="name=fetcher_"