Dockerization, whitenoise serving static, refactor

2025-04-04 10:53:16 +02:00
parent 5addfa5ba9
commit 4dbe2e55ef
39 changed files with 708 additions and 1238 deletions
--- a/app_urls/fetcher/tasks.py
+++ b/app_urls/fetcher/tasks.py
@@ -0,0 +1,114 @@
+from scheduler import job
+
+from .src.fetch_feed import FetchFeeds
+from .src.fetch_parser import FetchParser
+from .src.fetch_search import FetchSearcher
+from .src.fetch_missing_kids import FetchMissingKids
+from .src.db_utils import DB_Handler
+
+from .src.logger import get_logger
+logger = get_logger()
+
+@job('default')
+def fetch_feeds():
+    task = "Fetch Feeds"
+    logger.info("Task triggered: {}".format(task))
+    FetchFeeds().run()
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def fetch_parser():
+    task = "Fetch Parser"
+    logger.info("Task triggered: {}".format(task))
+    FetchParser().run()
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def fetch_search():
+    task = "Fetch Search"
+    logger.info("Task triggered: {}".format(task))
+    FetchSearcher().run()
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def fetch_missing_kids(number_pages=5):
+    task = "Fetch MissingKids"
+    logger.info("Task triggered: {}".format(task))
+    FetchMissingKids().run(number_pages)
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def fetch_missing_kids_all(number_pages=-1):
+    task = "Fetch MissingKids"
+    logger.info("Task triggered: {}".format(task))
+    FetchMissingKids().run(number_pages)
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def process_raw_urls(batch_size=50):
+    task = "Process raw URLs"
+    logger.info("Task triggered: {}".format(task))
+    DB_Handler().process_raw_urls(batch_size=batch_size)
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def process_error_urls(batch_size=50):
+    task = "Process error URLs"
+    logger.info("Task triggered: {}".format(task))
+    DB_Handler().process_error_urls(batch_size=batch_size)
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def process_missing_kids_urls(batch_size=50):
+    task = "Process Missing Kids URLs"
+    logger.info("Task triggered: {}".format(task))
+    DB_Handler().process_missing_kids_urls(batch_size=batch_size)
+    logger.info("Task completed: {}".format(task))
+
+@job('default')
+def process_missing_kids_urls_all(batch_size=None):
+    task = "Process Missing Kids URLs ALL"
+    logger.info("Task triggered: {}".format(task))
+    DB_Handler().process_missing_kids_urls(batch_size=batch_size)
+    logger.info("Task completed: {}".format(task))
+
+
+@job('default')
+def background_task(process_type: str):
+    logger.info("Task triggered: {}".format(process_type))
+
+    try:
+        if (process_type == "fetch_feeds"):
+            FetchFeeds().run()
+        elif (process_type == "fetch_parser"):
+            FetchParser().run()
+        elif (process_type == "fetch_search"):
+            FetchSearcher().run()
+        elif (process_type == "fetch_missingkids_all"):
+            FetchMissingKids().run(number_pages=-1)
+        elif ("fetch_missingkids" in process_type):
+            # number_pages encoded in URL
+            try:
+                number_pages = int(process_type.split("_")[-1])
+            except Exception as e:
+                number_pages = -1
+            FetchMissingKids().run(number_pages=number_pages)
+        elif ("process_" in process_type):
+            # Batch size encoded in URL
+            try:
+                batch_size = int(process_type.split("_")[-1])
+            except Exception as e:
+                batch_size = None
+            # Task type
+            if ("process_raw_urls" in process_type):
+                DB_Handler().process_raw_urls(batch_size=batch_size)
+            elif ("process_error_urls" in process_type):
+                DB_Handler().process_error_urls(batch_size=batch_size)
+            elif ("process_missing_kids_urls" in process_type):
+                DB_Handler().process_missing_kids_urls(batch_size=batch_size)
+        else:
+            logger.info("Task unknown!: {}".format(process_type))
+
+        logger.info("Task completed: {}".format(process_type))
+    except Exception as e:
+        logger.error(e)