Docker and deployment to fetcher server

app_urls/fetcher/src/fetch_search.py

@@ -54,6 +54,7 @@ class FetchSearcher():
                 for SearchInstance in ListSearchInstances:
                     # Sleep between requests, avoid too many requests...
                     time.sleep(float(os.getenv("FETCHER_BETWEEN_SEARCHES_SLEEP", 5)))
+                    # TODO: Random proxy / VPN
                     SearchInstance(args).fetch_articles(db_writer, obj_search)
 
                 # TODO: https://github.com/tasos-py/Search-Engines-Scraper/tree/master

app_urls/fetcher/src/fetch_search_instances.py

@@ -1,8 +1,6 @@
 import time
 import feedparser
 import os
-from django.utils import timezone
-from datetime import timedelta
 from ..models import Search, Source
 from .fetch_utils_gnews import decode_gnews_urls
 from .logger import get_logger

app_urls/fetcher/src/logger.py

@@ -11,7 +11,7 @@ logging.basicConfig(format='%(filename)s | %(levelname)s | %(asctime)s | %(messa
 logger = logging.getLogger("fetcher")
 logger.setLevel(logging.DEBUG)
 
-# To file log: INFO / WARNING / ERROR / CRITICAL
+# To file log: DEBUG / INFO / WARNING / ERROR / CRITICAL
 fh = logging.handlers.RotatingFileHandler(filename=os.path.join(logs_directory, "debug.log"), mode="a", maxBytes=10000000, backupCount=1)
 fh.setFormatter(logging.Formatter('%(levelname)s | %(asctime)s | %(message)s'))
 fh.setLevel(logging.DEBUG)