From 02f756d3c21a79a8000462d55a7741b4a0e138ba Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Thu, 14 Aug 2025 10:59:19 +0200 Subject: [PATCH] Ride missing kids exception --- app_urls/fetcher/src/db_utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app_urls/fetcher/src/db_utils.py b/app_urls/fetcher/src/db_utils.py index e2e2087..3b13956 100644 --- a/app_urls/fetcher/src/db_utils.py +++ b/app_urls/fetcher/src/db_utils.py @@ -106,8 +106,18 @@ class DB_Handler(): if ("missingkids.org" in obj_url.url) and ("poster" in obj_url.url): # Sleep required? To avoid too many requests error (original URL, not paywall bypassing endpoint) url_host_slowdown(obj_url.url, url_host_slowdown_seconds=float(os.getenv("FETCHER_URL_HOST_SLEEP", 5))) - # Request - r = requests.get(obj_url.url, allow_redirects=True) + try: + # Request + r = requests.get(obj_url.url, allow_redirects=True) + except Exception as e: + if (raise_exception_on_error): + # Simply raise exception, handled in a different way + raise Exception("Error processing URL, raising exception as expected") + else: + logger.debug("Error processing URL: {}\n{}\n{}".format(obj_url.url, str(e), traceback.format_exc())) + # Set status to error + self._set_status(obj_url, Urls.STATUS_ENUM.ERROR) + return if (r.url != obj_url.url): # Canonical