120 lines
3.8 KiB
YAML
120 lines
3.8 KiB
YAML
version: '3.9'
|
|
|
|
services:
|
|
|
|
fetcher_app_selenium:
|
|
image: fetcher_app_selenium
|
|
build:
|
|
context: ./app_selenium
|
|
args:
|
|
- ARCH=${ARCH} # arm64, amd64
|
|
container_name: fetcher_app_selenium
|
|
restart: unless-stopped
|
|
shm_size: 512mb
|
|
environment:
|
|
- SELENIUM_SLEEP_PER_PAGE=${SELENIUM_SLEEP_PER_PAGE}
|
|
- PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY}
|
|
ports:
|
|
- 80
|
|
dns:
|
|
- 1.1.1.1
|
|
- 1.0.0.1
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: '${DEPLOY_CPUS}'
|
|
memory: ${DEPLOY_RAM}
|
|
|
|
fetcher_app_urls:
|
|
image: fetcher_app_urls
|
|
build:
|
|
context: ./app_urls
|
|
container_name: fetcher_app_urls
|
|
restart: unless-stopped
|
|
environment:
|
|
# Initialization
|
|
- INITIALIZE_DB=${INITIALIZE_DB} # Related to DB persistence
|
|
- DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME}
|
|
- DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}
|
|
- DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL}
|
|
# Django
|
|
- DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} # host1,host2
|
|
- DJANGO_ALLOWED_ORIGINS=${DJANGO_ALLOWED_ORIGINS} # Reverse proxy
|
|
- DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY}
|
|
- DJANGO_DEBUG=${DJANGO_DEBUG}
|
|
- PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY}
|
|
# Database
|
|
- DB_NAME=${DB_NAME}
|
|
- DB_USER=${DB_USER}
|
|
- DB_PASSWORD=${DB_PASSWORD}
|
|
- DB_HOST=${DB_HOST}
|
|
- DB_PORT=${DB_PORT}
|
|
- REDIS_HOST=${REDIS_HOST}
|
|
- REDIS_PORT=${REDIS_PORT}
|
|
# Job timeout: 30 min
|
|
- JOB_DEFAULT_TIMEOUT=${JOB_DEFAULT_TIMEOUT}
|
|
# Fetcher
|
|
- FETCHER_GNEWS_DECODE_SLEEP=${FETCHER_GNEWS_DECODE_SLEEP}
|
|
- FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP=${FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP}
|
|
- FETCHER_BETWEEN_SEARCHES_SLEEP=${FETCHER_BETWEEN_SEARCHES_SLEEP} # Sleep time between each search
|
|
- FETCHER_URL_HOST_SLEEP=${FETCHER_URL_HOST_SLEEP} # Sleep time between requests to same URL host
|
|
- FETCHER_LANGUAGE_DETECTION_MIN_CHAR=${FETCHER_LANGUAGE_DETECTION_MIN_CHAR} # Min amonut of characters to run language detection
|
|
- FETCHER_INSERT_URL_CACHE_TIME=${FETCHER_INSERT_URL_CACHE_TIME} # Cache time: Insert raw URL
|
|
- FETCHER_ERROR_URL_CACHE_TIME=${FETCHER_ERROR_URL_CACHE_TIME} # Cache time: Error on processing URL
|
|
# Selenium
|
|
- SELENIUM_ENDPOINT=${SELENIUM_ENDPOINT}
|
|
- ENDPOINT_OLLAMA=${ENDPOINT_OLLAMA}
|
|
########################
|
|
#volumes: # Development mode
|
|
# - ./app_urls:/opt/app
|
|
########################
|
|
ports:
|
|
- 8000 # :8000
|
|
depends_on:
|
|
- fetcher_db
|
|
- fetcher_redis
|
|
dns:
|
|
- 1.1.1.1
|
|
- 1.0.0.1
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: '${DEPLOY_CPUS}'
|
|
memory: ${DEPLOY_RAM}
|
|
labels: # Reverse proxy sample
|
|
- "traefik.enable=true"
|
|
- "traefik.http.routers.fetcher.rule=Host(`${REVERSE_PROXY_URL}`)"
|
|
- "traefik.http.routers.fetcher.entrypoints=websecure"
|
|
- "traefik.http.routers.fetcher.tls.certresolver=myresolvercd"
|
|
- "traefik.http.services.fetcher.loadbalancer.server.port=8000"
|
|
networks:
|
|
- default # This network
|
|
- docker_default # Reverse proxy network
|
|
|
|
fetcher_db:
|
|
image: postgres:17
|
|
container_name: fetcher_db
|
|
restart: unless-stopped
|
|
# Set shared memory limit when using docker-compose
|
|
shm_size: 128mb
|
|
environment:
|
|
POSTGRES_DB: ${DB_NAME}
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD}
|
|
POSTGRES_USER: ${DB_USER}
|
|
POSTGRES_INITDB_ARGS: '--data-checksums'
|
|
volumes: # Persistent DB?
|
|
- ${PATH_DB_DATA}/postgres:/var/lib/postgresql/data
|
|
ports:
|
|
- 5432 #:5432
|
|
|
|
fetcher_redis:
|
|
image: redis:alpine
|
|
container_name: fetcher_redis
|
|
restart: unless-stopped
|
|
ports:
|
|
- 6379 #:6379
|
|
|
|
networks:
|
|
docker_default:
|
|
external: true
|