from celery import shared_task from .src.fetch_feed import FetchFeeds from .src.fetch_parser import FetchParser from .src.fetch_search import FetchSearcher from .src.fetch_missing_kids import FetchMissingKids from .src.fetch_selenium import FetchSeleniumSourceSearch from .src.db_utils import DB_Handler from .src.publisher import Publisher from .src.notifier import notify_telegram from .src.logger import get_logger logger = get_logger() @shared_task(queue='light') def process_raw_urls(batch_size=100): task = "Process raw URLs" logger.info("Task triggered: {}".format(task)) DB_Handler().process_raw_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @shared_task(queue='default') def process_error_urls(batch_size=50): task = "Process error URLs" logger.info("Task triggered: {}".format(task)) DB_Handler().process_error_urls(batch_size=batch_size) logger.info("Task completed: {}".format(task)) @shared_task(queue='light') def fetch_feeds(): task = "Fetch Feeds" logger.info("Task triggered: {}".format(task)) FetchFeeds().run() logger.info("Task completed: {}".format(task)) @shared_task(queue='default') def fetch_parser(): task = "Fetch Parser" logger.info("Task triggered: {}".format(task)) FetchParser().run() logger.info("Task completed: {}".format(task)) @shared_task(queue='default') def fetch_search(): task = "Fetch Search" logger.info("Task triggered: {}".format(task)) FetchSearcher().run() logger.info("Task completed: {}".format(task)) @shared_task(queue='heavy') def fetch_selenium_search(): task = "Fetch Selenium search" logger.info("Task triggered: {}".format(task)) FetchSeleniumSourceSearch().run() logger.info("Task completed: {}".format(task)) @shared_task(queue='heavy') def fetch_missing_kids(number_pages=5): task = "Fetch MissingKids" logger.info("Task triggered: {}".format(task)) FetchMissingKids().run(number_pages) logger.info("Task completed: {}".format(task)) @shared_task(queue='heavy') def process_missing_kids_urls(batch_size=None, process_status_only=None): task = "Process Missing Kids URLs - batch_size={} process_status_only={}".format(batch_size, process_status_only) logger.info("Task triggered: {}".format(task)) DB_Handler().process_missing_kids_urls(batch_size=batch_size, process_status_only=process_status_only) logger.info("Task completed: {}".format(task)) @shared_task(queue='default') def clean_old_url_content(older_than_days=14): task = "Clean old URL content" logger.info("Task triggered: {}".format(task)) DB_Handler().clean_old_url_content(older_than_days=older_than_days) logger.info("Task completed: {}".format(task)) @shared_task(queue='light') def notify_status(): task = "Notify status" logger.info("Task triggered: {}".format(task)) notify_telegram() logger.info("Task completed: {}".format(task))