Docker and deployment to fetcher server

2025-06-27 09:14:44 +02:00
parent f659d4adb3
commit 8b689729bf
12 changed files with 148 additions and 222 deletions
--- a/app_urls/fetcher/src/fetch_search.py
+++ b/app_urls/fetcher/src/fetch_search.py
@@ -54,6 +54,7 @@ class FetchSearcher():
                for SearchInstance in ListSearchInstances:
                    # Sleep between requests, avoid too many requests...
                    time.sleep(float(os.getenv("FETCHER_BETWEEN_SEARCHES_SLEEP", 5)))
+                    # TODO: Random proxy / VPN
                    SearchInstance(args).fetch_articles(db_writer, obj_search)

                # TODO: https://github.com/tasos-py/Search-Engines-Scraper/tree/master