matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MissingKids selenium app fix

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-07 12:38:30 +02:00
parent 3867538826
commit d78ade600f
4 changed files with 20 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app_selenium/Dockerfile
View File

@@ -1,9 +1,10 @@
FROM python:3.12 FROM python:3.12
# Architecture # Architecture: amd64
#ARG ARCH_G=linux64 #ARG ARCH_G=linux64
#ARG ARCH_F=linux-x86_64 #ARG ARCH_F=linux-x86_64
# Architecture: arm64
ARG ARCH_G=linux-aarch64 ARG ARCH_G=linux-aarch64
ARG ARCH_F=linux-aarch64 ARG ARCH_F=linux-aarch64

16
app_selenium/missing_kids.py
View File

@@ -8,7 +8,6 @@ import os
from logger import get_logger from logger import get_logger
logger = get_logger() logger = get_logger()
def get_webdriver(): def get_webdriver():
options = Options() options = Options()
options.add_argument('--headless') # Optional options.add_argument('--headless') # Optional
@@ -62,7 +61,7 @@ class MissingKidsFetcher():
num_exceptions += 1 num_exceptions += 1
processed_ok = False processed_ok = False
if (num_exceptions == 3): if (num_exceptions == 2):
continue_iterating = False continue_iterating = False
else: else:
logger.info("Exception while clicking page {}, retrying...".format(i+1)) logger.info("Exception while clicking page {}, retrying...".format(i+1))
@@ -77,22 +76,15 @@ class MissingKidsFetcher():
logger.info(e.text) logger.info(e.text)
# driver.refresh() # driver.refresh()
time.sleep(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4)); time.sleep(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4))
if (i == first_n_pages): if (i == first_n_pages):
continue_iterating = False continue_iterating = False
if (processed_ok): if (processed_ok):
i += 1 i += 1
num_exceptions = 0 num_exceptions = 0
except Exception as e: except Exception as e:
logger.warning("Exception while clicking page {}. {}".format(i+1, str(e)), exc_info=True) logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True)
finally: set_urls = set()
try:
logger.info("Closing web driver and returning results")
driver.close()
except Exception as e:
logger.warning("Exception closing web driver: {}".format(str(e)))
pass
return set_urls return set_urls

7
docker-compose-dev.yml
View File

@@ -8,6 +8,13 @@ services:
image: fetcher_app_selenium image: fetcher_app_selenium
build: build:
context: ./app_selenium context: ./app_selenium
args:
# arm64
#ARG ARCH_G=linux-aarch64
#ARG ARCH_F=linux-aarch64
# amd64
ARG ARCH_G=linux64
ARG ARCH_F=linux-x86_64
container_name: fetcher_app_selenium container_name: fetcher_app_selenium
restart: unless-stopped restart: unless-stopped
shm_size: 512mb shm_size: 512mb

7
docker-compose.yml
View File

@@ -6,6 +6,13 @@ services:
image: fetcher_app_selenium image: fetcher_app_selenium
build: build:
context: ./app_selenium context: ./app_selenium
args:
# arm64
ARG ARCH_G=linux-aarch64
ARG ARCH_F=linux-aarch64
# amd64
#ARG ARCH_G=linux64
#ARG ARCH_F=linux-x86_64
container_name: fetcher_app_selenium container_name: fetcher_app_selenium
restart: unless-stopped restart: unless-stopped
shm_size: 512mb shm_size: 512mb