matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Request timeout

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-09-05 13:52:34 +02:00
parent 35f9260b94
commit 2fae0a3a9d
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app_urls/fetcher/src/fetch_utils_url_processor.py
View File

@@ -58,7 +58,7 @@ def process_url(url, paywall_bypass=False):
# Process # Process
if ("foxnews.com" in url_of_interest) or ("zerohedge" in url_of_interest): if ("foxnews.com" in url_of_interest) or ("zerohedge" in url_of_interest):
# Request # Request
r = requests.get(url, headers={"User-Agent": user_agent}) r = requests.get(url, headers={"User-Agent": user_agent}, timeout=15)
# Raise for error code # Raise for error code
r.raise_for_status() r.raise_for_status()
# Parse # Parse
@@ -67,6 +67,7 @@ def process_url(url, paywall_bypass=False):
# Config: Fake user agent # Config: Fake user agent
config = newspaper.configuration.Configuration() config = newspaper.configuration.Configuration()
config.headers = {'User-Agent': user_agent} config.headers = {'User-Agent': user_agent}
config.request_timeout = 15 # timeout in seconds
# Default mode # Default mode
article = newspaper.article(url_of_interest, config=config) article = newspaper.article(url_of_interest, config=config)
except newspaper.ArticleBinaryDataException: except newspaper.ArticleBinaryDataException: