92 lines
3.8 KiB
Python
92 lines
3.8 KiB
Python
import src.credentials as cred
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s')
|
|
logger = logging.getLogger("news_fetcher")
|
|
logger.setLevel(logging.INFO)
|
|
|
|
import os
|
|
os.makedirs("logs", exist_ok=True)
|
|
|
|
# To file log
|
|
fh = RotatingFileHandler(filename="logs/log_app_fetcher.log", mode="a", maxBytes=10000000, backupCount=4)
|
|
fh.setFormatter(logging.Formatter('%(levelname)s | %(asctime)s | %(message)s'))
|
|
logger.addHandler(fh)
|
|
|
|
# To file log: WARNING / ERROR
|
|
fh_ = RotatingFileHandler(filename="logs/log_app_fetcher_error.log", mode="a", maxBytes=10000000, backupCount=1)
|
|
fh_.setFormatter(logging.Formatter('%(levelname)s | %(asctime)s | %(message)s'))
|
|
fh_.setLevel(logging.WARNING)
|
|
logger.addHandler(fh_)
|
|
|
|
logger.info("Environment: {}".format(cred.ENVIRONMENT))
|
|
|
|
##################################################################################################
|
|
from src.news_feed import NewsFeed
|
|
from src.news_parsing import NewsSiteParsing
|
|
from src.news_search import NewsSearch
|
|
from src.news_missing_kids import NewsMissingKids
|
|
from src.missing_kids_status import MissingKidsStatus
|
|
from src.url_status import UpdateErrorURLs
|
|
from src.fetcher_status import FetcherStatus
|
|
|
|
from fastapi import FastAPI, BackgroundTasks
|
|
# import requests
|
|
# from fastapi_utils.tasks import repeat_every
|
|
# import time
|
|
# time.sleep(10)
|
|
# import gc
|
|
|
|
app = FastAPI()
|
|
|
|
@app.get("/")
|
|
def hello_world():
|
|
return {"message": "OK"}
|
|
|
|
@app.get("/{fetch_type}")
|
|
async def fetch(background_tasks: BackgroundTasks, fetch_type: str):
|
|
# Concurrent job running
|
|
logger.info("Triggered fetch: {}".format(fetch_type))
|
|
|
|
if (fetch_type == "feeds"):
|
|
task_run = NewsFeed(cred.db_connect_info, cred.redis_connect_info).run
|
|
elif (fetch_type == "parser"):
|
|
task_run = NewsSiteParsing(cred.db_connect_info, cred.redis_connect_info).run
|
|
elif (fetch_type == "fetch_missing_kids_reduced"):
|
|
task_run = NewsMissingKids(cred.db_connect_info, cred.redis_connect_info, num_pages=4).run
|
|
elif (fetch_type == "fetch_missing_kids_full"):
|
|
task_run = NewsMissingKids(cred.db_connect_info, cred.redis_connect_info, num_pages=100000).run
|
|
elif (fetch_type == "search") or (fetch_type == "search_full"):
|
|
task_run = NewsSearch(cred.db_connect_info, cred.redis_connect_info, full=True).run
|
|
elif (fetch_type == "search_reduced"):
|
|
task_run = NewsSearch(cred.db_connect_info, cred.redis_connect_info, full=False).run
|
|
elif (fetch_type == "update_missing_kids_status_reduced"):
|
|
task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status
|
|
elif (fetch_type == "update_missing_kids_status_full"):
|
|
task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status
|
|
elif (fetch_type == "update_error_urls"):
|
|
task_run = UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status
|
|
elif (fetch_type == "fetch_warning_check"):
|
|
task_run = FetcherStatus(cred.db_connect_info, cred.redis_connect_info, last_minutes_check=180).check_warning
|
|
else:
|
|
return {"message": "ERROR. Unknown fetcher type!"}
|
|
|
|
# Run task
|
|
background_tasks.add_task(task_run)
|
|
# Return message
|
|
return {"message": "Started fetching {}: Ok".format(fetch_type)}
|
|
|
|
##################################################################################################
|
|
|
|
###########################
|
|
'''
|
|
@app.on_event("startup")
|
|
def verify_db() -> None:
|
|
logger.info("Testing DB connection")
|
|
import psycopg
|
|
with psycopg.connect(cred.db_connect_info) as conn:
|
|
url_test_msg = "Num URLs: {}".format(conn.execute("SELECT COUNT(*) FROM URLS;").fetchall())
|
|
logger.info(url_test_msg)
|
|
'''
|
|
###########################
|