matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tasks scheduler, news views, Obsolete code cleaning

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-03-21 16:13:56 +01:00
parent 8050773906
commit 0c6b5f1ea4
27 changed files with 1051 additions and 1669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
app_urls/api/tasks.py
View File

@@ -1,11 +1,11 @@
from django_rq import job
# from django_rq import job
from scheduler import job
from .src.fetch_feed import FetchFeeds
from .src.fetch_parser import FetchParser
from .src.fetch_search import FetchSearcher
from .src.db_utils import DB_Handler
'''
from src.fetch_search import FetchSearcher
from src.missing_kids_fetch import MissingKidsFetch
from src.missing_kids_status import MissingKidsStatus
'''
@@ -13,12 +13,56 @@ from src.missing_kids_status import MissingKidsStatus
from .src.logger import get_logger
logger = get_logger()
@job
@job('default')
def fetch_feeds():
logger.info("Task triggered: {}".format("FetchFeeds"))
task = "Fetch Feeds"
logger.info("Task triggered: {}".format(task))
FetchFeeds().run()
logger.info("Task completed: {}".format(task))
@job
@job('default')
def fetch_parser():
task = "Fetch Parser"
logger.info("Task triggered: {}".format(task))
FetchParser().run()
logger.info("Task completed: {}".format(task))
@job('default')
def fetch_search():
task = "Fetch Search"
logger.info("Task triggered: {}".format(task))
FetchSearcher().run()
logger.info("Task completed: {}".format(task))
# TODO: fetch_missing_kids()
@job('default')
def process_raw_urls(batch_size=50):
task = "Process raw URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_raw_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@job('default')
def process_error_urls(batch_size=50):
task = "Process error URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_error_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@job('default')
def process_missing_kids_urls(batch_size=50):
task = "Process Missing Kids URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_missing_kids_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@job('default')
def background_task(process_type: str):
logger.info("Task triggered: {}".format(process_type))
@@ -46,21 +90,11 @@ def background_task(process_type: str):
'''
elif (process_type == "search") or (process_type == "search_full"):
FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=True).run()
elif (process_type == "search_reduced"):
FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=False).run()
# Selenium based
elif (process_type == "fetch_missing_kids_reduced"):
MissingKidsFetch(db_handler, num_pages=4).run()
elif (process_type == "fetch_missing_kids_full"):
MissingKidsFetch(db_handler, num_pages=100000).run()
else:
logger.error("Task error, unknown type: {}".format(process_type))
return
'''
logger.info("Task completed: {}".format(process_type))