From 296a8fe8a8b5e6291ab8392d868b1fbfc993f56e Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Mon, 7 Apr 2025 19:12:36 +0200 Subject: [PATCH] Scheduled tasks timeout, selenium memory leak possible fix --- app_selenium/logger.py | 5 ++--- app_selenium/missing_kids.py | 6 +++++- app_urls/fetcher/views.py | 2 +- app_urls/scheduled_tasks.json | 6 +++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app_selenium/logger.py b/app_selenium/logger.py index a432466..c6413e6 100644 --- a/app_selenium/logger.py +++ b/app_selenium/logger.py @@ -8,9 +8,8 @@ logs_directory = os.getenv("PATH_LOGS_DIRECTORY", "logs") os.makedirs(logs_directory, exist_ok=True) # Too many logging entries otherwise -logging.getLogger("remote_connection.py").setLevel(logging.WARNING) -logging.getLogger("remote_connection").setLevel(logging.WARNING) - +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s') logger = logging.getLogger("selenium") diff --git a/app_selenium/missing_kids.py b/app_selenium/missing_kids.py index 2e852a4..e9aeba7 100644 --- a/app_selenium/missing_kids.py +++ b/app_selenium/missing_kids.py @@ -18,7 +18,6 @@ def get_webdriver(): driver = webdriver.Firefox(options=options, service=service) return driver - class MissingKidsFetcher(): def __init__(self) -> None: pass @@ -87,4 +86,9 @@ class MissingKidsFetcher(): logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True) set_urls = set() + try: + driver.close() + except Exception as e: + logger.warning("Exception while closing driver: {}".format(str(e)), exc_info=True) + return set_urls diff --git a/app_urls/fetcher/views.py b/app_urls/fetcher/views.py index bc9598b..e359e64 100644 --- a/app_urls/fetcher/views.py +++ b/app_urls/fetcher/views.py @@ -25,7 +25,7 @@ def link_list(request): # List of links list_links = \ [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \ - [ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "error"] ] + \ + [ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "warning"] ] + \ [ os.path.join(app_url, "task", l) for l in links_fetch + links_process ] # Json return JsonResponse({"links": list_links }) diff --git a/app_urls/scheduled_tasks.json b/app_urls/scheduled_tasks.json index 3a63536..02cf29b 100644 --- a/app_urls/scheduled_tasks.json +++ b/app_urls/scheduled_tasks.json @@ -72,7 +72,7 @@ "queue": "default", "repeat": null, "at_front": false, - "timeout": 1800, + "timeout": 3600, "result_ttl": 86400, "cron_string": null, "scheduled_time": "2025-01-01T00:00:00+00:00", @@ -135,7 +135,7 @@ "queue": "default", "repeat": null, "at_front": false, - "timeout": 1800, + "timeout": 3600, "result_ttl": 86400, "cron_string": null, "scheduled_time": "2025-01-01T00:00:00+00:00", @@ -177,7 +177,7 @@ "queue": "default", "repeat": null, "at_front": false, - "timeout": 1800, + "timeout": 3600, "result_ttl": 86400, "cron_string": null, "scheduled_time": "2025-01-01T00:00:00+00:00",