matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean old url content, use django db connection

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-04-14 10:48:33 +02:00
parent 0cd84496cf
commit 43c6c3aabf
5 changed files with 80 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app_urls/fetcher/tasks.py
View File

@@ -45,7 +45,7 @@ def fetch_missing_kids_all(number_pages=-1):
logger.info("Task completed: {}".format(task))
@job('default')
def process_raw_urls(batch_size=50):
def process_raw_urls(batch_size=100):
task = "Process raw URLs"
logger.info("Task triggered: {}".format(task))
DB_Handler().process_raw_urls(batch_size=batch_size)
@@ -72,6 +72,13 @@ def process_missing_kids_urls_all(batch_size=None):
DB_Handler().process_missing_kids_urls(batch_size=batch_size)
logger.info("Task completed: {}".format(task))
@job('default')
def clean_old_url_content(older_than_days=60):
task = "Clean old URL content"
logger.info("Task triggered: {}".format(task))
DB_Handler().clean_old_url_content(older_than_days=older_than_days)
logger.info("Task completed: {}".format(task))
@job('default')
def background_task(process_type: str):
@@ -86,6 +93,7 @@ def background_task(process_type: str):
FetchSearcher().run()
elif (process_type == "fetch_missingkids_all"):
FetchMissingKids().run(number_pages=-1)
elif ("fetch_missingkids" in process_type):
# number_pages encoded in URL
try:
@@ -93,6 +101,7 @@ def background_task(process_type: str):
except Exception as e:
number_pages = -1
FetchMissingKids().run(number_pages=number_pages)
elif ("process_" in process_type):
# Batch size encoded in URL
try:
@@ -106,6 +115,15 @@ def background_task(process_type: str):
DB_Handler().process_error_urls(batch_size=batch_size)
elif ("process_missing_kids_urls" in process_type):
DB_Handler().process_missing_kids_urls(batch_size=batch_size)
elif ( "clean_old_url_content" in process_type ):
# Older than X days encoded in URL
try:
older_than_days = float(process_type.split("_")[-1])
except Exception as e:
older_than_days = None
DB_Handler().clean_old_url_content(older_than_days=older_than_days)
else:
logger.info("Task unknown!: {}".format(process_type))