85 lines
2.9 KiB
Python
85 lines
2.9 KiB
Python
from celery import shared_task
|
|
|
|
from .src.fetch_feed import FetchFeeds
|
|
from .src.fetch_parser import FetchParser
|
|
from .src.fetch_search import FetchSearcher
|
|
from .src.fetch_missing_kids import FetchMissingKids
|
|
from .src.fetch_selenium import FetchSeleniumSourceSearch
|
|
from .src.db_utils import DB_Handler
|
|
from .src.publisher import Publisher
|
|
from .src.notifier import notify_telegram
|
|
|
|
from .src.logger import get_logger
|
|
logger = get_logger()
|
|
|
|
|
|
@shared_task(queue='default')
|
|
def fetch_feeds():
|
|
task = "Fetch Feeds"
|
|
logger.info("Task triggered: {}".format(task))
|
|
FetchFeeds().run()
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def fetch_parser():
|
|
task = "Fetch Parser"
|
|
logger.info("Task triggered: {}".format(task))
|
|
FetchParser().run()
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def fetch_search():
|
|
task = "Fetch Search"
|
|
logger.info("Task triggered: {}".format(task))
|
|
FetchSearcher().run()
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='low')
|
|
def fetch_selenium_search():
|
|
task = "Fetch Selenium search"
|
|
logger.info("Task triggered: {}".format(task))
|
|
FetchSeleniumSourceSearch().run()
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='low')
|
|
def fetch_missing_kids(number_pages=5):
|
|
task = "Fetch MissingKids"
|
|
logger.info("Task triggered: {}".format(task))
|
|
FetchMissingKids().run(number_pages)
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def process_raw_urls(batch_size=100):
|
|
task = "Process raw URLs"
|
|
logger.info("Task triggered: {}".format(task))
|
|
DB_Handler().process_raw_urls(batch_size=batch_size)
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def process_error_urls(batch_size=50):
|
|
task = "Process error URLs"
|
|
logger.info("Task triggered: {}".format(task))
|
|
DB_Handler().process_error_urls(batch_size=batch_size)
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='low')
|
|
def process_missing_kids_urls(batch_size=None, process_status_only=None):
|
|
task = "Process Missing Kids URLs - batch_size={} process_status_only={}".format(batch_size, process_status_only)
|
|
logger.info("Task triggered: {}".format(task))
|
|
DB_Handler().process_missing_kids_urls(batch_size=batch_size, process_status_only=process_status_only)
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def clean_old_url_content(older_than_days=14):
|
|
task = "Clean old URL content"
|
|
logger.info("Task triggered: {}".format(task))
|
|
DB_Handler().clean_old_url_content(older_than_days=older_than_days)
|
|
logger.info("Task completed: {}".format(task))
|
|
|
|
@shared_task(queue='default')
|
|
def notify_status():
|
|
task = "Notify status"
|
|
logger.info("Task triggered: {}".format(task))
|
|
notify_telegram()
|
|
logger.info("Task completed: {}".format(task))
|