79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
from src.fetch_feed import FetchFeed
|
|
from src.fetch_parser import FetchParser
|
|
from src.fetch_search import FetchSearch
|
|
|
|
from src.missing_kids_fetch import MissingKidsFetch
|
|
from src.missing_kids_status import MissingKidsStatus
|
|
|
|
from src.url_status import UpdateErrorURLs
|
|
from src.db_utils import DB_Handler
|
|
|
|
import src.credentials as cred
|
|
from logging_ import get_logger
|
|
|
|
from fastapi import FastAPI, BackgroundTasks
|
|
##################################################################################################
|
|
|
|
logger = get_logger()
|
|
logger.info("Environment: {}".format(cred.ENVIRONMENT))
|
|
|
|
db_handler = DB_Handler(cred.db_connect_info, cred.redis_connect_info)
|
|
|
|
app = FastAPI()
|
|
|
|
@app.get("/")
|
|
def hello_world():
|
|
return {"message": "Ok"}
|
|
|
|
@app.get("/{process_type}")
|
|
async def process(background_tasks: BackgroundTasks, process_type: str):
|
|
# Concurrent job running
|
|
logger.info("Triggered: {}".format(process_type))
|
|
|
|
if (process_type == "fetch_feeds"):
|
|
task_run = FetchFeed(db_handler).run
|
|
elif (process_type == "fetch_parser"):
|
|
task_run = FetchParser(db_handler).run
|
|
elif (process_type == "search") or (process_type == "search_full"):
|
|
task_run = FetchSearch(cred.db_connect_info, cred.redis_connect_info, full=True).run
|
|
elif (process_type == "search_reduced"):
|
|
task_run = FetchSearch(cred.db_connect_info, cred.redis_connect_info, full=False).run
|
|
|
|
# Selenium based
|
|
elif (process_type == "fetch_missing_kids_reduced"):
|
|
task_run = MissingKidsFetch(db_handler, num_pages=4).run
|
|
elif (process_type == "fetch_missing_kids_full"):
|
|
task_run = MissingKidsFetch(db_handler, num_pages=100000).run
|
|
|
|
elif (process_type == "update_missing_kids_status_reduced"):
|
|
task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status
|
|
elif (process_type == "update_missing_kids_status_full"):
|
|
task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status
|
|
|
|
elif (process_type == "update_error_urls"):
|
|
task_run = UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status
|
|
else:
|
|
return {"message": "ERROR. Unknown fetcher type!"}
|
|
|
|
# Run task
|
|
background_tasks.add_task(task_run)
|
|
# Return message
|
|
return {"message": "Started {}: Ok".format(process_type)}
|
|
|
|
"""
|
|
# TODO: Instead of background tasks!
|
|
|
|
import rq
|
|
import redis
|
|
|
|
# Redis connection
|
|
redis_conn = redis.Redis(host='localhost', port=6379, db=0)
|
|
queue = rq.Queue(connection=redis_conn)
|
|
|
|
# ...
|
|
# Queue the processing task
|
|
dict_args= {"db_handler": db_handler, }
|
|
queue.enqueue(task_run, **dict_args)
|
|
|
|
# https://python-rq.org/
|
|
""" |