Ride missing kids exception

app_urls/fetcher/src/db_utils.py

@@ -106,8 +106,18 @@ class DB_Handler():
         if ("missingkids.org" in obj_url.url) and ("poster" in obj_url.url):
             # Sleep required? To avoid too many requests error (original URL, not paywall bypassing endpoint)
             url_host_slowdown(obj_url.url, url_host_slowdown_seconds=float(os.getenv("FETCHER_URL_HOST_SLEEP", 5)))
-            # Request
-            r = requests.get(obj_url.url, allow_redirects=True)
+            try:
+                # Request
+                r = requests.get(obj_url.url, allow_redirects=True)
+            except Exception as e:
+                if (raise_exception_on_error):
+                    # Simply raise exception, handled in a different way
+                    raise Exception("Error processing URL, raising exception as expected")
+                else:
+                    logger.debug("Error processing URL: {}\n{}\n{}".format(obj_url.url, str(e), traceback.format_exc()))
+                    # Set status to error
+                    self._set_status(obj_url, Urls.STATUS_ENUM.ERROR)
+                    return
 
             if (r.url != obj_url.url):
                 # Canonical