import src.credentials as cred import logging from logging.handlers import RotatingFileHandler logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s') logger = logging.getLogger("news_fetcher") logger.setLevel(logging.INFO) import os os.makedirs("logs", exist_ok=True) # To file log fh = RotatingFileHandler(filename="logs/log_app_fetcher.log", mode="a", maxBytes=10000000, backupCount=4) fh.setFormatter(logging.Formatter('%(levelname)s | %(asctime)s | %(message)s')) logger.addHandler(fh) # To file log: WARNING / ERROR fh_ = RotatingFileHandler(filename="logs/log_app_fetcher_error.log", mode="a", maxBytes=10000000, backupCount=1) fh_.setFormatter(logging.Formatter('%(levelname)s | %(asctime)s | %(message)s')) fh_.setLevel(logging.WARNING) logger.addHandler(fh_) logger.info("Environment: {}".format(cred.ENVIRONMENT)) ################################################################################################## from src.news_feed import NewsFeed from src.news_parsing import NewsSiteParsing from src.news_search import NewsSearch from src.news_missing_kids import NewsMissingKids from src.missing_kids_status import MissingKidsStatus from src.url_status import UpdateErrorURLs from src.fetcher_status import FetcherStatus from fastapi import FastAPI, BackgroundTasks # import requests # from fastapi_utils.tasks import repeat_every # import time # time.sleep(10) # import gc app = FastAPI() @app.get("/") def hello_world(): return {"message": "OK"} @app.get("/{fetch_type}") async def fetch(background_tasks: BackgroundTasks, fetch_type: str): # Concurrent job running logger.info("Triggered fetch: {}".format(fetch_type)) if (fetch_type == "feeds"): task_run = NewsFeed(cred.db_connect_info, cred.redis_connect_info).run elif (fetch_type == "parser"): task_run = NewsSiteParsing(cred.db_connect_info, cred.redis_connect_info).run elif (fetch_type == "fetch_missing_kids_reduced"): task_run = NewsMissingKids(cred.db_connect_info, cred.redis_connect_info, num_pages=4).run elif (fetch_type == "fetch_missing_kids_full"): task_run = NewsMissingKids(cred.db_connect_info, cred.redis_connect_info, num_pages=100000).run elif (fetch_type == "search") or (fetch_type == "search_full"): task_run = NewsSearch(cred.db_connect_info, cred.redis_connect_info, full=True).run elif (fetch_type == "search_reduced"): task_run = NewsSearch(cred.db_connect_info, cred.redis_connect_info, full=False).run elif (fetch_type == "update_missing_kids_status_reduced"): task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=50).update_missing_kids_status elif (fetch_type == "update_missing_kids_status_full"): task_run = MissingKidsStatus(cred.db_connect_info, cred.redis_connect_info, num_urls=None).update_missing_kids_status elif (fetch_type == "update_error_urls"): task_run = UpdateErrorURLs(cred.db_connect_info, cred.redis_connect_info, num_urls=100).update_error_urls_status elif (fetch_type == "fetch_warning_check"): task_run = FetcherStatus(cred.db_connect_info, cred.redis_connect_info, last_minutes_check=180).check_warning else: return {"message": "ERROR. Unknown fetcher type!"} # Run task background_tasks.add_task(task_run) # Return message return {"message": "Started fetching {}: Ok".format(fetch_type)} ################################################################################################## ########################### ''' @app.on_event("startup") def verify_db() -> None: logger.info("Testing DB connection") import psycopg with psycopg.connect(cred.db_connect_info) as conn: url_test_msg = "Num URLs: {}".format(conn.execute("SELECT COUNT(*) FROM URLS;").fetchall()) logger.info(url_test_msg) ''' ###########################