from scheduler import job from .src.fetch_feed import FetchFeeds from .src.fetch_parser import FetchParser from .src.fetch_search import FetchSearcher from .src.fetch_missing_kids import FetchMissingKids from .src.db_utils import DB_Handler from .src.publisher import Publisher from .src.logger import get_logger logger = get_logger() @job('default') def fetch_feeds(): task = "Fetch Feeds" logger.info("Task triggered: {}".format(task)) FetchFeeds().run() logger.info("Task completed: {}".format(task)) @job('default') def fetch_parser(): task = "Fetch Parser" logger.info("Task triggered: {}".format(task)) FetchParser().run() logger.info("Task completed: {}".format(task)) @job('default') def fetch_search(): task = "Fetch Search" logger.info("Task triggered: {}".format(task)) FetchSearcher().run() logger.info("Task completed: {}".format(task)) @job('default') def fetch_missing_kids(number_pages=5): task = "Fetch MissingKids" logger.info("Task triggered: {}".format(task)) FetchMissingKids().run(number_pages) logger.info("Task completed: {}".format(task)) @job('default') def fetch_missing_kids_all(number_pages=-1): task = "Fetch MissingKids" logger.info("Task triggered: {}".format(task)) FetchMissingKids().run(number_pages) logger.info("Task completed: {}".format(task)) @job('default') def process_raw_urls(batch_size=100): task = "Process raw URLs" logger.info("Task triggered: {}".format(task)) DB_Handler().process_raw_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @job('default') def process_error_urls(batch_size=50): task = "Process error URLs" logger.info("Task triggered: {}".format(task)) DB_Handler().process_error_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @job('default') def process_missing_kids_urls(batch_size=50): task = "Process Missing Kids URLs" logger.info("Task triggered: {}".format(task)) DB_Handler().process_missing_kids_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @job('default') def process_missing_kids_urls_all(batch_size=None): task = "Process Missing Kids URLs ALL" logger.info("Task triggered: {}".format(task)) DB_Handler().process_missing_kids_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @job('default') def clean_old_url_content(older_than_days=14): task = "Clean old URL content" logger.info("Task triggered: {}".format(task)) DB_Handler().clean_old_url_content(older_than_days=older_than_days) logger.info("Task completed: {}".format(task)) @job('default') def background_task(process_type: str): logger.info("Task triggered: {}".format(process_type)) try: if (process_type == "fetch_feeds"): FetchFeeds().run() elif (process_type == "fetch_parser"): FetchParser().run() elif (process_type == "fetch_search"): FetchSearcher().run() elif (process_type == "fetch_missingkids_all"): FetchMissingKids().run(number_pages=-1) elif ("fetch_missingkids" in process_type): # number_pages encoded in URL try: number_pages = int(process_type.split("_")[-1]) except Exception as e: number_pages = -1 FetchMissingKids().run(number_pages=number_pages) elif ("process_" in process_type): # Batch size encoded in URL try: batch_size = int(process_type.split("_")[-1]) except Exception as e: batch_size = None # Task type if ("process_raw_urls" in process_type): DB_Handler().process_raw_urls(batch_size=batch_size) elif ("process_error_urls" in process_type): DB_Handler().process_error_urls(batch_size=batch_size) elif ("process_missing_kids_urls" in process_type): DB_Handler().process_missing_kids_urls(batch_size=batch_size) elif ("clean_old_url_content" in process_type ): # Older than X days encoded in URL try: older_than_days = float(process_type.split("_")[-1]) except Exception as e: older_than_days = None DB_Handler().clean_old_url_content(older_than_days=older_than_days) elif ("publish" in process_type): # Extract URL ID url_id = process_type.split("_")[-1] # Publish Publisher().publish(url_id) else: logger.info("Task unknown!: {}".format(process_type)) logger.info("Task completed: {}".format(process_type)) except Exception as e: logger.error(e)