from django_rq import job from .src.fetch_feed import FetchFeeds from .src.fetch_parser import FetchParser from .src.db_utils import DB_Handler ''' from src.fetch_search import FetchSearcher from src.missing_kids_fetch import MissingKidsFetch from src.missing_kids_status import MissingKidsStatus ''' from .src.logger import get_logger logger = get_logger() @job def background_task(process_type: str): logger.info("Task triggered: {}".format(process_type)) try: if (process_type == "fetch_feeds"): FetchFeeds().run() elif (process_type == "fetch_parser"): FetchParser().run() # TODO: ENCODE BATCH_SIZE IN PROCESS_tYPE.. elif (process_type == "process_raw_urls"): DB_Handler().process_raw_urls(batch_size=50) elif (process_type == "process_error_urls"): DB_Handler().process_error_urls(batch_size=50) elif (process_type == "process_missing_kids_urls"): DB_Handler().process_missing_kids_urls(batch_size=50) elif ("process_missing_kids_urls" in process_type): batch_size = int(process_type.split("_")[-1]) DB_Handler().process_missing_kids_urls(batch_size=batch_size) else: logger.info("Task unknown!: {}".format(process_type)) ''' elif (process_type == "search") or (process_type == "search_full"): FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=True).run() elif (process_type == "search_reduced"): FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=False).run() # Selenium based elif (process_type == "fetch_missing_kids_reduced"): MissingKidsFetch(db_handler, num_pages=4).run() elif (process_type == "fetch_missing_kids_full"): MissingKidsFetch(db_handler, num_pages=100000).run() elif (process_type == "update_missing_kids_status_reduced"): MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status() elif (process_type == "update_missing_kids_status_full"): MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status() elif (process_type == "update_error_urls"): UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status() else: logger.error("Task error, unknown type: {}".format(process_type)) return ''' logger.info("Task completed: {}".format(process_type)) except Exception as e: logger.error(e)