Working fetch feeds and process raw urls

This commit is contained in:
Luciano Gervasoni
2025-03-13 18:23:28 +01:00
parent 61c31ee9aa
commit 7d7bce1e72
11 changed files with 318 additions and 136 deletions

View File

@@ -15,18 +15,20 @@ from src.credentials import db_connect_info, redis_connect_info
db_handler = DB_Handler(db_connect_info, redis_connect_info)
'''
import logging
logger = logging.getLogger(__name__)
from .src.logger import get_logger
logger = get_logger()
@job
def background_task(process_type: str):
logger.info("Task triggered: {}".format(process_type))
try:
FetchFeeds().run()
# DB_Handler().process_raw_urls()
if (process_type == "fetch_feeds"):
FetchFeeds().run()
elif (process_type == "process_raw_urls"):
DB_Handler().process_raw_urls(batch_size=3)
else:
logger.info("Task unknown!: {}".format(process_type))
'''