matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

General search fix, status pattern match regex, find feeds on startup

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-09 15:52:35 +02:00
parent 296a8fe8a8
commit f369b23d81
22 changed files with 538 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app_selenium/logger.py
View File

@@ -7,12 +7,8 @@ logs_directory = os.getenv("PATH_LOGS_DIRECTORY", "logs")
# Directory of logs
os.makedirs(logs_directory, exist_ok=True)
# Too many logging entries otherwise
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s')
logger = logging.getLogger("selenium")
logger = logging.getLogger("app_selenium")
logger.setLevel(logging.DEBUG)
# To file log: INFO / WARNING / ERROR / CRITICAL

5
app_selenium/missing_kids.py
View File

@@ -39,7 +39,7 @@ class MissingKidsFetcher():
logger.debug("Processing page: {}...".format(i))
try:
time.sleep(int(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4))) #driver.implicitly_wait(3)
time.sleep(float(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4))) #driver.implicitly_wait(3)
# Fetch poster URLs
for element_type in ["a"]: # ["a", "p", "div"]:
for elem in driver.find_elements(By.TAG_NAME, element_type):
@@ -75,7 +75,7 @@ class MissingKidsFetcher():
logger.info(e.text)
# driver.refresh()
time.sleep(int(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4)))
time.sleep(float(os.getenv("SELENIUM_SLEEP_PER_PAGE", 4)))
if (i == first_n_pages):
continue_iterating = False
@@ -86,6 +86,7 @@ class MissingKidsFetcher():
logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True)
set_urls = set()
# Release memory
try:
driver.close()
except Exception as e: