matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Workers light,default,heavy

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-09-08 12:34:47 +02:00
parent 892fb984d1
commit 2ed86e31ec
4 changed files with 48 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
app_urls/fetcher/tasks.py
View File

@@ -13,13 +13,30 @@ from .src.logger import get_logger
logger = get_logger()
@shared_task(queue='light')
def process_raw_urls(batch_size=100):
task = "Process raw URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_raw_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
def process_error_urls(batch_size=50):
task = "Process error URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_error_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@shared_task(queue='light')
def fetch_feeds():
task = "Fetch Feeds"
logger.info("Task triggered: {}".format(task))
FetchFeeds().run()
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
def fetch_parser():
task = "Fetch Parser"
@@ -34,41 +51,31 @@ def fetch_search():
FetchSearcher().run()
logger.info("Task completed: {}".format(task))
@shared_task(queue='low')
@shared_task(queue='heavy')
def fetch_selenium_search():
task = "Fetch Selenium search"
logger.info("Task triggered: {}".format(task))
FetchSeleniumSourceSearch().run()
logger.info("Task completed: {}".format(task))
@shared_task(queue='low')
@shared_task(queue='heavy')
def fetch_missing_kids(number_pages=5):
task = "Fetch MissingKids"
logger.info("Task triggered: {}".format(task))
FetchMissingKids().run(number_pages)
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
def process_raw_urls(batch_size=100):
task = "Process raw URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_raw_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
def process_error_urls(batch_size=50):
task = "Process error URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_error_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@shared_task(queue='low')
@shared_task(queue='heavy')
def process_missing_kids_urls(batch_size=None, process_status_only=None):
task = "Process Missing Kids URLs - batch_size={} process_status_only={}".format(batch_size, process_status_only)
logger.info("Task triggered: {}".format(task))
DB_Handler().process_missing_kids_urls(batch_size=batch_size, process_status_only=process_status_only)
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
def clean_old_url_content(older_than_days=14):
task = "Clean old URL content"
@@ -76,7 +83,7 @@ def clean_old_url_content(older_than_days=14):
DB_Handler().clean_old_url_content(older_than_days=older_than_days)
logger.info("Task completed: {}".format(task))
@shared_task(queue='default')
@shared_task(queue='light')
def notify_status():
task = "Notify status"
logger.info("Task triggered: {}".format(task))