68 lines
2.5 KiB
Python
68 lines
2.5 KiB
Python
from django_rq import job
|
|
|
|
from .src.fetch_feed import FetchFeeds
|
|
from .src.db_utils import DB_Handler
|
|
'''
|
|
from src.fetch_parser import FetchParser
|
|
from src.fetch_search import FetchSearcher
|
|
from src.missing_kids_fetch import MissingKidsFetch
|
|
from src.missing_kids_status import MissingKidsStatus
|
|
from src.url_status import UpdateErrorURLs
|
|
from src.db_utils import DB_Handler
|
|
from src.credentials import db_connect_info, redis_connect_info
|
|
|
|
# DB Handler
|
|
db_handler = DB_Handler(db_connect_info, redis_connect_info)
|
|
'''
|
|
|
|
from .src.logger import get_logger
|
|
logger = get_logger()
|
|
|
|
@job
|
|
def background_task(process_type: str):
|
|
logger.info("Task triggered: {}".format(process_type))
|
|
|
|
try:
|
|
if (process_type == "fetch_feeds"):
|
|
FetchFeeds().run()
|
|
elif (process_type == "process_raw_urls"):
|
|
DB_Handler().process_raw_urls(batch_size=3)
|
|
else:
|
|
logger.info("Task unknown!: {}".format(process_type))
|
|
|
|
|
|
'''
|
|
if (process_type == "fetch_feeds"):
|
|
FetchFeeds(db_handler).run()
|
|
|
|
elif (process_type == "fetch_parser"):
|
|
FetchParser(db_handler).run()
|
|
|
|
elif (process_type == "search") or (process_type == "search_full"):
|
|
FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=True).run()
|
|
elif (process_type == "search_reduced"):
|
|
FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=False).run()
|
|
|
|
# Selenium based
|
|
elif (process_type == "fetch_missing_kids_reduced"):
|
|
MissingKidsFetch(db_handler, num_pages=4).run()
|
|
elif (process_type == "fetch_missing_kids_full"):
|
|
MissingKidsFetch(db_handler, num_pages=100000).run()
|
|
|
|
elif (process_type == "update_missing_kids_status_reduced"):
|
|
MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status()
|
|
elif (process_type == "update_missing_kids_status_full"):
|
|
MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status()
|
|
|
|
elif (process_type == "update_error_urls"):
|
|
UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status()
|
|
|
|
else:
|
|
logger.error("Task error, unknown type: {}".format(process_type))
|
|
return
|
|
'''
|
|
|
|
logger.info("Task completed: {}".format(process_type))
|
|
except Exception as e:
|
|
logger.error(e)
|