matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Url visualize fix, setting job timeout, decode news.google.com debug

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-06 20:31:05 +02:00
parent b55243009a
commit 4011717c34
3 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app_urls/fetcher/src/fetch_utils.py
View File

@@ -1,4 +1,5 @@
import os
import time
from django.core.cache import cache
from .logger import get_logger
logger = get_logger()
@@ -28,8 +29,11 @@ def decode_gnews_urls(encoded_urls, interval=int(os.getenv("FETCHER_GNEWS_DECODE
list_decoded_urls.append(decoded_url)
# Cache decoded URL
cache.set("gnews_decode_{}".format(url), decoded_url, timeout=60*60*12)
elif ("429 Client Error: Too Many Requests for url" in decoded_url_dict.get("message")):
logger.debug("Decoding news.google.com - 429 Too many requests: {}".format(decoded_url_dict.get("message")))
# time.sleep()
else:
logger.info("Bad status while decoding news.google.com, URL {}\n{}".format(url, decoded_url_dict.get("message")))
logger.info("Decoding news.google.com - Bad status for URL {}\n{}".format(url, decoded_url_dict.get("message")))
except Exception as e:
logger.warning("Error decoding news.google.com, URL: {}".format(url))
return list_decoded_urls