Working fetch search, refactoring DB towards source search

This commit is contained in:
Luciano Gervasoni
2025-03-20 11:42:33 +01:00
parent 83f76232b2
commit 05e17266f1
14 changed files with 558 additions and 120 deletions

View File

@@ -1,5 +1,5 @@
from .db_utils import DB_Handler
from ..models import WebsiteOfInterest
from ..models import Search
import newspaper
import traceback
from .logger import get_logger
@@ -14,7 +14,7 @@ class FetchParser():
logger.debug("Starting FetchParser.run() for {}")
# Get URL hosts
list_url_host = list(WebsiteOfInterest.objects.values_list('url_host', flat=True))
list_url_host = list(Search.objects.filter(type=Search.TYPE_ENUM.URL_HOST).values_list('search', flat=True))
logger.debug("Fetching news by parsing URL hosts: {}".format(list_url_host))
# Process newspaper4k build method