Working fetch feeds and parser, process raw and error urls

OBSOLETE_app_fetcher/app.py

@@ -0,0 +1,79 @@
+from src.fetch_feed import FetchFeed
+from src.fetch_parser import FetchParser
+from src.fetch_search import FetchSearch
+
+from src.missing_kids_fetch import MissingKidsFetch
+from src.missing_kids_status import MissingKidsStatus
+
+from src.url_status import UpdateErrorURLs
+from src.db_utils import DB_Handler
+
+import src.credentials as cred
+from logging_ import get_logger
+
+from fastapi import FastAPI, BackgroundTasks
+##################################################################################################
+
+logger = get_logger()
+logger.info("Environment: {}".format(cred.ENVIRONMENT))
+
+db_handler = DB_Handler(cred.db_connect_info, cred.redis_connect_info)
+
+app = FastAPI()
+
+@app.get("/")
+def hello_world():
+    return {"message": "Ok"}
+
+@app.get("/{process_type}")
+async def process(background_tasks: BackgroundTasks, process_type: str):
+    # Concurrent job running
+    logger.info("Triggered: {}".format(process_type))
+
+    if (process_type == "fetch_feeds"):
+        task_run = FetchFeed(db_handler).run
+    elif (process_type == "fetch_parser"):
+        task_run = FetchParser(db_handler).run
+    elif (process_type == "search") or (process_type == "search_full"):
+        task_run = FetchSearch(cred.db_connect_info, cred.redis_connect_info, full=True).run
+    elif (process_type == "search_reduced"):
+        task_run = FetchSearch(cred.db_connect_info, cred.redis_connect_info, full=False).run
+
+    # Selenium based
+    elif (process_type == "fetch_missing_kids_reduced"):
+        task_run = MissingKidsFetch(db_handler, num_pages=4).run
+    elif (process_type == "fetch_missing_kids_full"):
+        task_run = MissingKidsFetch(db_handler, num_pages=100000).run
+    
+    elif (process_type == "update_missing_kids_status_reduced"):
+        task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status
+    elif (process_type == "update_missing_kids_status_full"):
+        task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status
+
+    elif (process_type == "update_error_urls"):
+        task_run = UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status
+    else:
+        return {"message": "ERROR. Unknown fetcher type!"}
+    
+    # Run task
+    background_tasks.add_task(task_run)
+    # Return message
+    return {"message": "Started {}: Ok".format(process_type)}
+
+"""
+# TODO: Instead of background tasks!
+
+import rq
+import redis
+
+# Redis connection
+redis_conn = redis.Redis(host='localhost', port=6379, db=0)
+queue = rq.Queue(connection=redis_conn)
+
+# ... 
+# Queue the processing task
+dict_args= {"db_handler": db_handler, }
+queue.enqueue(task_run, **dict_args)
+
+# https://python-rq.org/
+"""