From dc56cc3365b5fd5e153f72af3c3ddf3369e82320 Mon Sep 17 00:00:00 2001 From: Matitos Date: Wed, 27 Aug 2025 12:55:00 +0200 Subject: [PATCH] Read only fetcher urls app --- app_urls/initialize.sh | 6 ++--- app_urls/supervisord.conf | 39 --------------------------- docker-compose-base.yml | 2 +- docker-compose.yml | 57 +++++++++++++++++++++++++-------------- 4 files changed, 41 insertions(+), 63 deletions(-) diff --git a/app_urls/initialize.sh b/app_urls/initialize.sh index ec8e3ed..e27cd22 100755 --- a/app_urls/initialize.sh +++ b/app_urls/initialize.sh @@ -4,12 +4,12 @@ if [ "${INITIALIZE_DB}" = false ]; then echo "Initialization not required" else echo "Initializating database" - python init_db.py --initialize_tables --initialize_data + # python init_db.py --initialize_tables --initialize_data python manage.py makemigrations fetcher; python manage.py migrate --fake-initial - python manage.py migrate django_celery_beat + # python manage.py migrate django_celery_beat python manage.py createsuperuser --noinput python manage.py collectstatic --no-input - python manage.py loaddata scheduled_tasks.json + # python manage.py loaddata scheduled_tasks.json # # python manage.py inspectdb # Debugging model fi diff --git a/app_urls/supervisord.conf b/app_urls/supervisord.conf index 9707793..edc9f48 100644 --- a/app_urls/supervisord.conf +++ b/app_urls/supervisord.conf @@ -13,42 +13,3 @@ redirect_stderr=true ; Rotate when file reaches max size stdout_logfile_maxbytes=20MB stdout_logfile_backups=1 - -[program:beat] -command=celery -A core beat -l info --logfile=/opt/logs/beat.log -directory=/opt/app -autostart=true -autorestart=true -; Unified log file -stdout_logfile=/opt/logs/beat.log -stderr_logfile=/opt/logs/beat.log -redirect_stderr=true -; Rotate when file reaches max size -stdout_logfile_maxbytes=20MB -stdout_logfile_backups=1 - -[program:worker_default] -command=celery -A core worker -l info --logfile=/opt/logs/worker_default.log --concurrency=1 -Q default -n default -directory=/opt/app -autostart=true -autorestart=true -; Unified log file -stdout_logfile=/opt/logs/worker_default.log -stderr_logfile=/opt/logs/worker_default.log -redirect_stderr=true -; Rotate when file reaches max size -stdout_logfile_maxbytes=20MB -stdout_logfile_backups=1 - -[program:worker_low] -command=celery -A core worker -l info --logfile=/opt/logs/worker_low.log --concurrency=1 -Q low -n low -directory=/opt/app -autostart=true -autorestart=true -; Unified log file -stdout_logfile=/opt/logs/worker_low.log -stderr_logfile=/opt/logs/worker_low.log -redirect_stderr=true -; Rotate when file reaches max size -stdout_logfile_maxbytes=20MB -stdout_logfile_backups=1 diff --git a/docker-compose-base.yml b/docker-compose-base.yml index c67a706..9aeecb6 100644 --- a/docker-compose-base.yml +++ b/docker-compose-base.yml @@ -72,7 +72,7 @@ services: depends_on: - fetcher_db - fetcher_redis - - fetcher_app_selenium + # - fetcher_app_selenium dns: - 1.1.1.1 - 1.0.0.1 diff --git a/docker-compose.yml b/docker-compose.yml index 51cc79f..685900b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,34 @@ services: - fetcher_app_selenium: - extends: - file: docker-compose-base.yml - service: fetcher_app_selenium - deploy: - resources: - limits: - cpus: '${DEPLOY_CPUS}' - memory: ${DEPLOY_RAM} + #fetcher_app_selenium: + # extends: + # file: docker-compose-base.yml + # service: fetcher_app_selenium + # deploy: + # resources: + # limits: + # cpus: '${DEPLOY_CPUS}' + # memory: ${DEPLOY_RAM} fetcher_app_urls: extends: file: docker-compose-base.yml service: fetcher_app_urls ports: - - 8000:8000 + - 8067:8000 deploy: resources: limits: cpus: '${DEPLOY_CPUS}' memory: ${DEPLOY_RAM} + labels: # Reverse proxy sample + - "traefik.enable=true" + - "traefik.http.routers.fetcher.rule=Host(`fetcher.matitos.org`)" + - "traefik.http.routers.fetcher.entrypoints=websecure" + - "traefik.http.routers.fetcher.tls.certresolver=myresolvercd" + - "traefik.http.services.fetcher.loadbalancer.server.port=8000" + networks: + - docker_default # Reverse proxy network fetcher_db: extends: @@ -36,16 +44,18 @@ services: # REQUIREMENT: Add fetcher's SSH public key into the DB's .ssh/authorized_keys machine - ~/.ssh:/root/.ssh:ro ports: - - 15885:15885 + - 15889:15889 - 5432:5432 command: - sh - -c - | apk add --update openssh autossh - # Monitor status on port 15885 - autossh -M 15885 -N -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} - # autossh -M 15885 -N -o 'GatewayPorts yes' -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} + # Monitor status on port 15889 + autossh -M 15889 -N -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} + # autossh -M 15889 -N -o 'GatewayPorts yes' -L 0.0.0.0:5432:127.0.0.1:5432 ${REMOTE_USERNAME}@${REMOTE_HOST} + networks: + - docker_default # Reverse proxy network fetcher_redis: extends: @@ -53,10 +63,17 @@ services: service: fetcher_redis ports: - 6379:6379 + networks: + - docker_default # Reverse proxy network - fetcher_flower: - extends: - file: docker-compose-base.yml - service: fetcher_flower - ports: - - 5555:5555 + #fetcher_flower: + # extends: + # file: docker-compose-base.yml + # service: fetcher_flower + # ports: + # - 5555:5555 + + +networks: + docker_default: + external: true