diff --git a/app_urls/fetcher/src/fetch_utils_url_processor.py b/app_urls/fetcher/src/fetch_utils_url_processor.py index 0805fd5..8d04b09 100644 --- a/app_urls/fetcher/src/fetch_utils_url_processor.py +++ b/app_urls/fetcher/src/fetch_utils_url_processor.py @@ -58,7 +58,7 @@ def process_url(url, paywall_bypass=False): # Process if ("foxnews.com" in url_of_interest) or ("zerohedge" in url_of_interest): # Request - r = requests.get(url, headers={"User-Agent": user_agent}) + r = requests.get(url, headers={"User-Agent": user_agent}, timeout=15) # Raise for error code r.raise_for_status() # Parse @@ -67,6 +67,7 @@ def process_url(url, paywall_bypass=False): # Config: Fake user agent config = newspaper.configuration.Configuration() config.headers = {'User-Agent': user_agent} + config.request_timeout = 15 # timeout in seconds # Default mode article = newspaper.article(url_of_interest, config=config) except newspaper.ArticleBinaryDataException: