matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Url visualize fix, setting job timeout, decode news.google.com debug

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-06 20:31:05 +02:00
parent b55243009a
commit 4011717c34
3 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app_urls/core/settings.py
View File

@@ -129,6 +129,7 @@ SCHEDULER_QUEUES = {
}
}
SCHEDULER_CONFIG = {
'DEFAULT_JOB_TIMEOUT': os.environ.get("JOB_DEFAULT_TIMEOUT", 60*30), # 30 minutes
'DEFAULT_TIMEOUT': os.environ.get("JOB_DEFAULT_TIMEOUT", 60*30), # 30 minutes
'DEFAULT_RESULT_TTL': 60*60*12, # 12 hours
'EXECUTIONS_IN_PAGE': 20,

6
app_urls/fetcher/src/fetch_utils.py
View File

@@ -1,4 +1,5 @@
import os
import time
from django.core.cache import cache
from .logger import get_logger
logger = get_logger()
@@ -28,8 +29,11 @@ def decode_gnews_urls(encoded_urls, interval=int(os.getenv("FETCHER_GNEWS_DECODE
list_decoded_urls.append(decoded_url)
# Cache decoded URL
cache.set("gnews_decode_{}".format(url), decoded_url, timeout=60*60*12)
elif ("429 Client Error: Too Many Requests for url" in decoded_url_dict.get("message")):
logger.debug("Decoding news.google.com - 429 Too many requests: {}".format(decoded_url_dict.get("message")))
# time.sleep()
else:
logger.info("Bad status while decoding news.google.com, URL {}\n{}".format(url, decoded_url_dict.get("message")))
logger.info("Decoding news.google.com - Bad status for URL {}\n{}".format(url, decoded_url_dict.get("message")))
except Exception as e:
logger.warning("Error decoding news.google.com, URL: {}".format(url))
return list_decoded_urls

2
app_urls/fetcher/templates/filtered_urls.html
View File

@@ -370,7 +370,7 @@ input[type="checkbox"] {
{% for url in urls %}
<tr>
<td><a href="./{{ url.id }}" class="btn btn-primary btn-sm" target="_blank">{{ url.id }}</a></td>
<td><a href="{{ url.url }}/" target="_blank">{{ url.url }}</a></td>
<td><a href="{{ url.url }}" target="_blank">{{ url.url }}</a></td>
<td>
{% if url.status == 'raw' %}
<span class="badge bg-secondary">{{ url.status|capfirst }}</span>