matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Error URLs proces with marreta. Env vars update

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-16 11:50:55 +02:00
parent b876f6d720
commit 148ec72658
6 changed files with 44 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
app_urls/fetcher/src/db_utils.py
View File

@@ -6,16 +6,14 @@ from django.utils import timezone
from datetime import timedelta
from .url_processor import process_url, get_with_protocol
import re
import os
import traceback
from .logger import get_logger
logger = get_logger()
class DB_Handler():
def __init__(self):
# Inserting raw URL, cache time: 1 day
self._cache_timeout_insert_url = 86400
# Processing error URL, cache time: 2 days
self._cache_timeout_error_url = 86400*2
pass
def insert_raw_urls(self, urls, obj_source, obj_search):
try:
@@ -75,8 +73,8 @@ class DB_Handler():
# Insert or update cache
for url in urls_clean:
cache.set("insert_{}".format(url), True, timeout=self._cache_timeout_insert_url)
cache.set("insert_{}{}{}".format(url, obj_source.source, obj_search.search), True, timeout=self._cache_timeout_insert_url)
cache.set("insert_{}".format(url), True, timeout=int(os.getenv("FETCHER_INSERT_URL_CACHE_TIME", 86400)))
cache.set("insert_{}{}{}".format(url, obj_source.source, obj_search.search), True, timeout=int(os.getenv("FETCHER_INSERT_URL_CACHE_TIME", 86400)))
logger.info("Inserted #{} raw URLs, Source-Search {} - {}".format(len(urls_to_insert), obj_source.source, obj_search.search))
@@ -84,7 +82,7 @@ class DB_Handler():
logger.warning("Exception inserting raw URLs: {}\n{}".format(e, traceback.format_exc()))
def _process_single_url(self, obj_url, status_pattern_match, raise_exception_on_error):
def _process_single_url(self, obj_url, status_pattern_match, raise_exception_on_error, override_url=None):
def set_status(obj_url, status):
# Update status if setting a new value
@@ -102,8 +100,13 @@ class DB_Handler():
return
try:
# Override URL for request?
if (override_url is not None):
url_of_interest = override_url
else:
url_of_interest = obj_url.url
# Extract URL content
dict_url_data = process_url(obj_url.url)
dict_url_data = process_url(url_of_interest)
except Exception as e:
if (raise_exception_on_error):
# Simply raise exception, handled in a different way
@@ -232,11 +235,12 @@ class DB_Handler():
try:
# Process URL
self._process_single_url(obj_url, status_pattern_match=None, raise_exception_on_error=True)
override_url = "https://marreta.pcdomanual.com/p/{}".format(obj_url.url)
self._process_single_url(obj_url, status_pattern_match=None, raise_exception_on_error=True, override_url=override_url)
num_urls_processed += 1
except Exception as e:
# Error, cache to avoid re-processing for X time
cache.set("error_{}".format(obj_url.id), True, timeout=self._cache_timeout_insert_url)
cache.set("error_{}".format(obj_url.id), True, timeout=int(os.getenv("FETCHER_ERROR_URL_CACHE_TIME", 86400)))
num_urls_skipped += 1
# Get following batch of URLs, status='error'