matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring fetcher WIP

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-03-07 11:52:35 +01:00
parent ec4a2cad15
commit 95b9766245
10 changed files with 124 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app_fetcher/app.py
View File

@@ -30,6 +30,8 @@ from src.missing_kids_status import MissingKidsStatus
from src.url_status import UpdateErrorURLs
from src.fetcher_status import FetcherStatus
from src.db_utils import DB_Handler
from fastapi import FastAPI, BackgroundTasks
# import requests
# from fastapi_utils.tasks import repeat_every
@@ -37,11 +39,13 @@ from fastapi import FastAPI, BackgroundTasks
# time.sleep(10)
# import gc
db_handler = DB_Handler(cred.db_connect_info, cred.redis_connect_info)
app = FastAPI()
@app.get("/")
def hello_world():
return {"message": "OK"}
return {"message": "Ok"}
@app.get("/{fetch_type}")
async def fetch(background_tasks: BackgroundTasks, fetch_type: str):
@@ -49,9 +53,9 @@ async def fetch(background_tasks: BackgroundTasks, fetch_type: str):
logger.info("Triggered fetch: {}".format(fetch_type))
if (fetch_type == "feeds"):
task_run = NewsFeed(cred.db_connect_info, cred.redis_connect_info).run
task_run = NewsFeed(db_handler).run
elif (fetch_type == "parser"):
task_run = NewsSiteParsing(cred.db_connect_info, cred.redis_connect_info).run
task_run = NewsSiteParsing(db_handler).run
elif (fetch_type == "fetch_missing_kids_reduced"):
task_run = NewsMissingKids(cred.db_connect_info, cred.redis_connect_info, num_pages=4).run
elif (fetch_type == "fetch_missing_kids_full"):