120 lines
4.1 KiB
YAML
120 lines
4.1 KiB
YAML
version: '3.9'
|
|
|
|
services:
|
|
|
|
fetcher_app_selenium:
|
|
image: fetcher_app_selenium
|
|
build:
|
|
context: ./app_selenium
|
|
args:
|
|
- ARCH=${ARCH} # arm64, amd64
|
|
container_name: fetcher_app_selenium
|
|
restart: unless-stopped
|
|
shm_size: 512mb
|
|
environment:
|
|
- SELENIUM_SLEEP_PER_PAGE=${SELENIUM_SLEEP_PER_PAGE}
|
|
- PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY}
|
|
ports:
|
|
- 80
|
|
dns:
|
|
- 1.1.1.1
|
|
- 1.0.0.1
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: '${DEPLOY_CPUS}'
|
|
memory: ${DEPLOY_RAM}
|
|
|
|
fetcher_app_urls:
|
|
image: fetcher_app_urls
|
|
build:
|
|
context: ./app_urls
|
|
container_name: fetcher_app_urls
|
|
restart: unless-stopped
|
|
environment:
|
|
# Initialization
|
|
- INITIALIZE_DB=${INITIALIZE_DB} # Related to DB persistence
|
|
- DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME}
|
|
- DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}
|
|
- DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL}
|
|
# Django
|
|
- DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} # host1,host2
|
|
- DJANGO_ALLOWED_ORIGINS=${DJANGO_ALLOWED_ORIGINS} # Reverse proxy
|
|
- DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY}
|
|
- DJANGO_DEBUG=${DJANGO_DEBUG}
|
|
- PATH_LOGS_DIRECTORY=${PATH_LOGS_DIRECTORY}
|
|
# Database
|
|
- DB_NAME=${DB_NAME}
|
|
- DB_USER=${DB_USER}
|
|
- DB_PASSWORD=${DB_PASSWORD}
|
|
- DB_HOST=${DB_HOST}
|
|
- DB_PORT=${DB_PORT}
|
|
- REDIS_HOST=${REDIS_HOST}
|
|
- REDIS_PORT=${REDIS_PORT}
|
|
# Job timeout: 30 min
|
|
- JOB_DEFAULT_TIMEOUT=${JOB_DEFAULT_TIMEOUT}
|
|
# Fetcher
|
|
- FETCHER_GNEWS_DECODE_SLEEP=${FETCHER_GNEWS_DECODE_SLEEP}
|
|
- FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP=${FETCHER_GOOGLE_GENERAL_PAGE_ITER_SLEEP}
|
|
- FETCHER_BETWEEN_SEARCHES_SLEEP=${FETCHER_BETWEEN_SEARCHES_SLEEP} # Sleep time between each search
|
|
- FETCHER_URL_HOST_SLEEP=${FETCHER_URL_HOST_SLEEP} # Sleep time between requests to same URL host
|
|
- FETCHER_LANGUAGE_DETECTION_MIN_CHAR=${FETCHER_LANGUAGE_DETECTION_MIN_CHAR} # Min amonut of characters to run language detection
|
|
- FETCHER_INSERT_URL_CACHE_TIME=${FETCHER_INSERT_URL_CACHE_TIME} # Cache time: Insert raw URL
|
|
- FETCHER_ERROR_URL_CACHE_TIME=${FETCHER_ERROR_URL_CACHE_TIME} # Cache time: Error on processing URL
|
|
# Selenium
|
|
- SELENIUM_ENDPOINT=${SELENIUM_ENDPOINT}
|
|
- ENDPOINT_OLLAMA=${ENDPOINT_OLLAMA}
|
|
# Ghost
|
|
- GHOST_ADMIN_API_KEY=${GHOST_ADMIN_API_KEY}
|
|
- GHOST_ADMIN_API_URL=${GHOST_ADMIN_API_URL}
|
|
- PEXELS_API_KEY=${PEXELS_API_KEY}
|
|
- OLLAMA_MODEL_DEFAULT=${OLLAMA_MODEL_DEFAULT}
|
|
########################
|
|
#volumes: # Development mode
|
|
# - ./app_urls:/opt/app
|
|
########################
|
|
ports:
|
|
- 8000:8000
|
|
depends_on:
|
|
- fetcher_db
|
|
- fetcher_redis
|
|
dns:
|
|
- 1.1.1.1
|
|
- 1.0.0.1
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
cpus: '${DEPLOY_CPUS}'
|
|
memory: ${DEPLOY_RAM}
|
|
|
|
fetcher_db:
|
|
container_name: fetcher_db
|
|
image: alpine:latest
|
|
restart: unless-stopped
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 256M
|
|
volumes:
|
|
# REQUIREMENT: Add fetcher's SSH public key into the DB's .ssh/authorized_keys machine
|
|
- ~/.ssh:/root/.ssh:ro
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
apk add --update openssh autossh
|
|
autossh -M 15885 -N -o 'GatewayPorts yes' -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST}
|
|
### Alternative:
|
|
### autossh -M 0 -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o 'GatewayPorts yes' -L 15882:127.0.0.1:15882 matitos@matitos.org
|
|
### -M 15882 monitors on port X, if already being used conflict!
|
|
###autossh -M 15882 -N -o 'GatewayPorts yes' -L 15882:127.0.0.1:15882 matitos@matitos.org
|
|
###ssh -N -o "StrictHostKeyChecking no" -o "ServerAliveInterval 60" -o "ServerAliveCountMax 3" -o 'PasswordAuthentication no' -o 'GatewayPorts yes' -L 15882:127.0.0.1:15882 matitos@matitos.org
|
|
network_mode: "host"
|
|
|
|
fetcher_redis:
|
|
image: redis:alpine
|
|
container_name: fetcher_redis
|
|
restart: unless-stopped
|
|
ports:
|
|
- 6379 #:6379
|