matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MissingKids selenium app fix

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-07 12:38:30 +02:00
parent 3867538826
commit d78ade600f
4 changed files with 20 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app_selenium/Dockerfile
View File

@@ -1,9 +1,10 @@
FROM python:3.12
# Architecture
# Architecture: amd64
#ARG ARCH_G=linux64
#ARG ARCH_F=linux-x86_64
# Architecture: arm64
ARG ARCH_G=linux-aarch64
ARG ARCH_F=linux-aarch64

16
app_selenium/missing_kids.py
View File

@@ -8,7 +8,6 @@ import os
from logger import get_logger
logger = get_logger()
def get_webdriver():
options = Options()
options.add_argument('--headless') # Optional
@@ -62,7 +61,7 @@ class MissingKidsFetcher():
num_exceptions += 1
processed_ok = False
if (num_exceptions == 3):
if (num_exceptions == 2):
continue_iterating = False
else:
logger.info("Exception while clicking page {}, retrying...".format(i+1))
@@ -77,22 +76,15 @@ class MissingKidsFetcher():
logger.info(e.text)
# driver.refresh()
time.sleep(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4));
time.sleep(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4))
if (i == first_n_pages):
continue_iterating = False
if (processed_ok):
i += 1
num_exceptions = 0
except Exception as e:
logger.warning("Exception while clicking page {}. {}".format(i+1, str(e)), exc_info=True)
finally:
try:
logger.info("Closing web driver and returning results")
driver.close()
except Exception as e:
logger.warning("Exception closing web driver: {}".format(str(e)))
pass
logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True)
set_urls = set()
return set_urls

7
docker-compose-dev.yml
View File

@@ -8,6 +8,13 @@ services:
image: fetcher_app_selenium
build:
context: ./app_selenium
args:
# arm64
#ARG ARCH_G=linux-aarch64
#ARG ARCH_F=linux-aarch64
# amd64
ARG ARCH_G=linux64
ARG ARCH_F=linux-x86_64
container_name: fetcher_app_selenium
restart: unless-stopped
shm_size: 512mb

7
docker-compose.yml
View File

@@ -6,6 +6,13 @@ services:
image: fetcher_app_selenium
build:
context: ./app_selenium
args:
# arm64
ARG ARCH_G=linux-aarch64
ARG ARCH_F=linux-aarch64
# amd64
#ARG ARCH_G=linux64
#ARG ARCH_F=linux-x86_64
container_name: fetcher_app_selenium
restart: unless-stopped
shm_size: 512mb