46 lines
1.9 KiB
Python
46 lines
1.9 KiB
Python
from .db_utils import DB_Handler
|
|
import newspaper
|
|
import logging
|
|
logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(message)s')
|
|
logger = logging.getLogger("news_fetcher")
|
|
|
|
class NewsSiteParsing():
|
|
def __init__(self, db_handler: DB_Handler) -> None:
|
|
logger.debug("Initializing News SiteParsing newspaper4k")
|
|
self.db_handler = db_handler
|
|
|
|
# TODO: MOVE LOGIC ELSEWHERE!
|
|
def _postprocess(self, article_urls):
|
|
return [url.replace("#comment-stream", "") for url in article_urls]
|
|
|
|
def run(self):
|
|
try:
|
|
logger.debug("Starting NewsSiteParsing.run() for {}")
|
|
|
|
# Get URL hosts
|
|
list_url_hosts = self.db_handler._get_url_hosts()
|
|
logger.info("Fetching news by parsing URL hosts: {}".format(str(list_url_hosts)))
|
|
|
|
# Process newspaper4k build method
|
|
for url_host_feed in list_url_hosts:
|
|
# Protocol
|
|
if not (url_host_feed.startswith("http")):
|
|
url_host_feed_formatted = "https://" + url_host_feed
|
|
else:
|
|
url_host_feed_formatted = url_host_feed
|
|
|
|
logger.debug("Fetching newspaper4k parsing based on URL: {}".format(url_host_feed_formatted))
|
|
# Source object
|
|
url_host_built = newspaper.build(url_host_feed_formatted)
|
|
# Get articles URL list
|
|
urls_fetched = url_host_built.article_urls()
|
|
# TODO: MOVE!
|
|
# Post-processing
|
|
urls_fetched = self._postprocess(urls_fetched)
|
|
|
|
# URL fetching source
|
|
source = "newspaper4k {}".format(url_host_feed)
|
|
# Write to DB
|
|
self.db_handler.write_batch(urls_fetched, source)
|
|
except Exception as e:
|
|
logger.warning("Exception in NewsSiteParsing.run(): {}".format(str(e))) |