matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Request timeout debugging

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-09-05 14:00:50 +02:00
parent 2fae0a3a9d
commit e4a325d6b4
2 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app_urls/fetcher/src/db_utils.py
View File

@@ -148,6 +148,7 @@ class DB_Handler():
try: try:
# Extract URL content # Extract URL content
dict_url_data = process_url(obj_url.url, paywall_bypass) dict_url_data = process_url(obj_url.url, paywall_bypass)
logger.debug("Processing raw URL EXTRACT URL CONTENT OK: {}".format(obj_url.url))
except Exception as e: except Exception as e:
if (raise_exception_on_error): if (raise_exception_on_error):
# Simply raise exception, handled in a different way # Simply raise exception, handled in a different way
@@ -242,6 +243,7 @@ class DB_Handler():
status_pattern_match = _get_status_pattern_matching(obj_url.url, list_pattern_status_tuple) status_pattern_match = _get_status_pattern_matching(obj_url.url, list_pattern_status_tuple)
# Process URL # Process URL
self._process_single_url(obj_url, status_pattern_match, raise_exception_on_error=False) self._process_single_url(obj_url, status_pattern_match, raise_exception_on_error=False)
logger.debug("Processing raw URL OK: {}".format(obj_url.url))
logger.info("Updated #{} raw URLs".format(len(raw_urls))) logger.info("Updated #{} raw URLs".format(len(raw_urls)))
except Exception as e: except Exception as e:

1
app_urls/fetcher/src/fetch_utils_url_processor.py
View File

@@ -40,6 +40,7 @@ def url_host_slowdown(url, url_host_slowdown_seconds):
cache.set("process_{}".format(url_host).encode("utf-8"), time.time(), timeout=60*5) # Expire after 5 minutes cache.set("process_{}".format(url_host).encode("utf-8"), time.time(), timeout=60*5) # Expire after 5 minutes
def process_url(url, paywall_bypass=False): def process_url(url, paywall_bypass=False):
logger.debug("Processing raw URL 1: {}".format(url))
if (paywall_bypass): if (paywall_bypass):
# TODO: Implement self-hosted instance # TODO: Implement self-hosted instance