matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scheduled tasks timeout, selenium memory leak possible fix

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-07 19:12:36 +02:00
parent d41fb26a99
commit 296a8fe8a8
4 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app_selenium/logger.py
View File

@@ -8,9 +8,8 @@ logs_directory = os.getenv("PATH_LOGS_DIRECTORY", "logs")
os.makedirs(logs_directory, exist_ok=True) os.makedirs(logs_directory, exist_ok=True)
# Too many logging entries otherwise # Too many logging entries otherwise
logging.getLogger("remote_connection.py").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("remote_connection").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s') logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s')
logger = logging.getLogger("selenium") logger = logging.getLogger("selenium")

6
app_selenium/missing_kids.py
View File

@@ -18,7 +18,6 @@ def get_webdriver():
driver = webdriver.Firefox(options=options, service=service) driver = webdriver.Firefox(options=options, service=service)
return driver return driver
class MissingKidsFetcher(): class MissingKidsFetcher():
def __init__(self) -> None: def __init__(self) -> None:
pass pass
@@ -87,4 +86,9 @@ class MissingKidsFetcher():
logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True) logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True)
set_urls = set() set_urls = set()
try:
driver.close()
except Exception as e:
logger.warning("Exception while closing driver: {}".format(str(e)), exc_info=True)
return set_urls return set_urls

2
app_urls/fetcher/views.py
View File

@@ -25,7 +25,7 @@ def link_list(request):
# List of links # List of links
list_links = \ list_links = \
[ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \ [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \
[ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "error"] ] + \ [ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "warning"] ] + \
[ os.path.join(app_url, "task", l) for l in links_fetch + links_process ] [ os.path.join(app_url, "task", l) for l in links_fetch + links_process ]
# Json # Json
return JsonResponse({"links": list_links }) return JsonResponse({"links": list_links })

6
app_urls/scheduled_tasks.json
View File

@@ -72,7 +72,7 @@
"queue": "default", "queue": "default",
"repeat": null, "repeat": null,
"at_front": false, "at_front": false,
"timeout": 1800, "timeout": 3600,
"result_ttl": 86400, "result_ttl": 86400,
"cron_string": null, "cron_string": null,
"scheduled_time": "2025-01-01T00:00:00+00:00", "scheduled_time": "2025-01-01T00:00:00+00:00",
@@ -135,7 +135,7 @@
"queue": "default", "queue": "default",
"repeat": null, "repeat": null,
"at_front": false, "at_front": false,
"timeout": 1800, "timeout": 3600,
"result_ttl": 86400, "result_ttl": 86400,
"cron_string": null, "cron_string": null,
"scheduled_time": "2025-01-01T00:00:00+00:00", "scheduled_time": "2025-01-01T00:00:00+00:00",
@@ -177,7 +177,7 @@
"queue": "default", "queue": "default",
"repeat": null, "repeat": null,
"at_front": false, "at_front": false,
"timeout": 1800, "timeout": 3600,
"result_ttl": 86400, "result_ttl": 86400,
"cron_string": null, "cron_string": null,
"scheduled_time": "2025-01-01T00:00:00+00:00", "scheduled_time": "2025-01-01T00:00:00+00:00",