Scheduled tasks timeout, selenium memory leak possible fix

app_selenium/logger.py

@@ -8,9 +8,8 @@ logs_directory = os.getenv("PATH_LOGS_DIRECTORY", "logs")
 os.makedirs(logs_directory, exist_ok=True)
 
 # Too many logging entries otherwise
-logging.getLogger("remote_connection.py").setLevel(logging.WARNING)
-logging.getLogger("remote_connection").setLevel(logging.WARNING)
-
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s')
 logger = logging.getLogger("selenium")

app_selenium/missing_kids.py

@@ -18,7 +18,6 @@ def get_webdriver():
     driver = webdriver.Firefox(options=options, service=service)
     return driver
 
-
 class MissingKidsFetcher():
     def __init__(self) -> None:
         pass
@@ -87,4 +86,9 @@ class MissingKidsFetcher():
             logger.warning("Exception while fetching MissingKids {}".format(str(e)), exc_info=True)
             set_urls = set()
 
+        try:
+            driver.close()
+        except Exception as e:
+            logger.warning("Exception while closing driver: {}".format(str(e)), exc_info=True)
+
         return set_urls

app_urls/fetcher/views.py

@@ -25,7 +25,7 @@ def link_list(request):
     # List of links
     list_links = \
         [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \
-        [ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "error"] ] + \
+        [ os.path.join(app_url, "logs", log_type) for log_type in ["debug", "info", "warning"] ] + \
         [ os.path.join(app_url, "task", l) for l in links_fetch + links_process ]
     # Json
     return JsonResponse({"links": list_links })

app_urls/scheduled_tasks.json

@@ -72,7 +72,7 @@
     "queue": "default",
     "repeat": null,
     "at_front": false,
-    "timeout": 1800,
+    "timeout": 3600,
     "result_ttl": 86400,
     "cron_string": null,
     "scheduled_time": "2025-01-01T00:00:00+00:00",
@@ -135,7 +135,7 @@
     "queue": "default",
     "repeat": null,
     "at_front": false,
-    "timeout": 1800,
+    "timeout": 3600,
     "result_ttl": 86400,
     "cron_string": null,
     "scheduled_time": "2025-01-01T00:00:00+00:00",
@@ -177,7 +177,7 @@
     "queue": "default",
     "repeat": null,
     "at_front": false,
-    "timeout": 1800,
+    "timeout": 3600,
     "result_ttl": 86400,
     "cron_string": null,
     "scheduled_time": "2025-01-01T00:00:00+00:00",