Working fetch feeds and parser, process raw and error urls

app_urls/api/tasks.py

@@ -1,6 +1,7 @@
 from django_rq import job
 
 from .src.fetch_feed import FetchFeeds
+from .src.fetch_parser import FetchParser
 from .src.db_utils import DB_Handler
 '''
 from src.fetch_parser import FetchParser
@@ -8,16 +9,13 @@ from src.fetch_search import FetchSearcher
 from src.missing_kids_fetch import MissingKidsFetch
 from src.missing_kids_status import MissingKidsStatus
 from src.url_status import UpdateErrorURLs
-from src.db_utils import DB_Handler
-from src.credentials import db_connect_info, redis_connect_info
-
-# DB Handler
-db_handler = DB_Handler(db_connect_info, redis_connect_info)
 '''
 
 from .src.logger import get_logger
 logger = get_logger()
 
+# TODO: Queues with priorities, process_raw_urls least priority due to slowdown...
+
 @job
 def background_task(process_type: str):
     logger.info("Task triggered: {}".format(process_type))
@@ -25,18 +23,17 @@ def background_task(process_type: str):
     try:
         if (process_type == "fetch_feeds"):
             FetchFeeds().run()
+        elif (process_type == "fetch_parser"):
+            FetchParser().run()
         elif (process_type == "process_raw_urls"):
-            DB_Handler().process_raw_urls(batch_size=3)
+            DB_Handler().process_raw_urls(batch_size=50)
+        elif (process_type == "process_error_urls"):
+            DB_Handler().process_error_urls(batch_size=50)
         else:
             logger.info("Task unknown!: {}".format(process_type))
 
         
         '''
-        if (process_type == "fetch_feeds"):
-            FetchFeeds(db_handler).run()
-            
-        elif (process_type == "fetch_parser"):
-            FetchParser(db_handler).run()
             
         elif (process_type == "search") or (process_type == "search_full"):
             FetchSearcher(cred.db_connect_info, cred.redis_connect_info, full=True).run()