diff --git a/app_urls/core/settings.py b/app_urls/core/settings.py index 7e19eac..307a65b 100644 --- a/app_urls/core/settings.py +++ b/app_urls/core/settings.py @@ -129,6 +129,7 @@ SCHEDULER_QUEUES = { } } SCHEDULER_CONFIG = { + 'DEFAULT_JOB_TIMEOUT': os.environ.get("JOB_DEFAULT_TIMEOUT", 60*30), # 30 minutes 'DEFAULT_TIMEOUT': os.environ.get("JOB_DEFAULT_TIMEOUT", 60*30), # 30 minutes 'DEFAULT_RESULT_TTL': 60*60*12, # 12 hours 'EXECUTIONS_IN_PAGE': 20, diff --git a/app_urls/fetcher/src/fetch_utils.py b/app_urls/fetcher/src/fetch_utils.py index 829ec35..d79841b 100644 --- a/app_urls/fetcher/src/fetch_utils.py +++ b/app_urls/fetcher/src/fetch_utils.py @@ -1,4 +1,5 @@ import os +import time from django.core.cache import cache from .logger import get_logger logger = get_logger() @@ -28,8 +29,11 @@ def decode_gnews_urls(encoded_urls, interval=int(os.getenv("FETCHER_GNEWS_DECODE list_decoded_urls.append(decoded_url) # Cache decoded URL cache.set("gnews_decode_{}".format(url), decoded_url, timeout=60*60*12) + elif ("429 Client Error: Too Many Requests for url" in decoded_url_dict.get("message")): + logger.debug("Decoding news.google.com - 429 Too many requests: {}".format(decoded_url_dict.get("message"))) + # time.sleep() else: - logger.info("Bad status while decoding news.google.com, URL {}\n{}".format(url, decoded_url_dict.get("message"))) + logger.info("Decoding news.google.com - Bad status for URL {}\n{}".format(url, decoded_url_dict.get("message"))) except Exception as e: logger.warning("Error decoding news.google.com, URL: {}".format(url)) return list_decoded_urls \ No newline at end of file diff --git a/app_urls/fetcher/templates/filtered_urls.html b/app_urls/fetcher/templates/filtered_urls.html index 3cb86fb..28859ce 100644 --- a/app_urls/fetcher/templates/filtered_urls.html +++ b/app_urls/fetcher/templates/filtered_urls.html @@ -370,7 +370,7 @@ input[type="checkbox"] { {% for url in urls %} {{ url.id }} - {{ url.url }} + {{ url.url }} {% if url.status == 'raw' %} {{ url.status|capfirst }}