From 3a33697ca3210a01e58c5e4d5b78abfe00da58ec Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Tue, 1 Sep 2026 15:08:14 -0300 Subject: [PATCH 1/5] Fix critical bugs in app_urls - init_db.py: parameterize SQL queries to prevent SQL injection - scheduled_tasks.json: resolve duplicate periodic task pk (4 -> 15) - db_utils.py: use function parameter in _get_status_pattern_matching instead of capturing the enclosing scope variable - url_detail.html: fix wrong field name image_urls -> images_url - settings.py: align Celery queues (default/light/heavy) with the task queues and supervisord workers - Add /task/publish_/ route to fix broken publish links in templates --- app_urls/core/settings.py | 3 ++- app_urls/fetcher/src/db_utils.py | 2 +- app_urls/fetcher/templates/url_detail.html | 2 +- app_urls/fetcher/urls.py | 1 + app_urls/fetcher/views.py | 12 +++++++++++- app_urls/init_db.py | 17 +++++++---------- app_urls/scheduled_tasks.json | 2 +- 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/app_urls/core/settings.py b/app_urls/core/settings.py index dd2effa..faa2dd6 100644 --- a/app_urls/core/settings.py +++ b/app_urls/core/settings.py @@ -123,7 +123,8 @@ CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' CELERY_TASK_QUEUES = ( Queue('default'), - Queue('low'), + Queue('light'), + Queue('heavy'), ) diff --git a/app_urls/fetcher/src/db_utils.py b/app_urls/fetcher/src/db_utils.py index d0ba365..eac056e 100644 --- a/app_urls/fetcher/src/db_utils.py +++ b/app_urls/fetcher/src/db_utils.py @@ -212,7 +212,7 @@ class DB_Handler(): # Sort pattern tuples by priority. (pattern, priority, status) for regex_pattern, regex_priority, status_if_match in sorted(list_pattern_status_tuple, key=lambda tup: tup[1], reverse=True): # Regular expression pattern matching: https://regexr.com/ - if bool(re.match(regex_pattern, obj_url.url)): + if bool(re.match(regex_pattern, url)): # logger.debug("Regex pattern found, status '{}' for URL: {}".format(status_if_match, url)) return status_if_match return None diff --git a/app_urls/fetcher/templates/url_detail.html b/app_urls/fetcher/templates/url_detail.html index 5360209..3515647 100644 --- a/app_urls/fetcher/templates/url_detail.html +++ b/app_urls/fetcher/templates/url_detail.html @@ -246,7 +246,7 @@ Image URLs - {{ url_content.image_urls|default:"" }} + {{ url_content.images_url|default:"" }} Video URLs diff --git a/app_urls/fetcher/urls.py b/app_urls/fetcher/urls.py index 69605fb..a70434d 100644 --- a/app_urls/fetcher/urls.py +++ b/app_urls/fetcher/urls.py @@ -17,4 +17,5 @@ urlpatterns = [ # path('urls/', views.filtered_urls, name='filtered_urls'), path('urls//', views.url_detail_view, name='url_detail'), + path('task/publish_/', views.publish, name='publish'), ] diff --git a/app_urls/fetcher/views.py b/app_urls/fetcher/views.py index be15d61..39c96b0 100644 --- a/app_urls/fetcher/views.py +++ b/app_urls/fetcher/views.py @@ -2,12 +2,13 @@ from .views_base import link_list, logs, log_db #, trigger_task, from django.core.paginator import Paginator from django.shortcuts import render, get_object_or_404 -from django.http import StreamingHttpResponse, JsonResponse +from django.http import StreamingHttpResponse, JsonResponse, HttpResponse from django.db.models import Q, Count from django.utils import timezone from django.utils.timezone import now, timedelta from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch, UrlsDuplicate from .src.llm import OllamaClient +from .src.publisher import Publisher import json @@ -62,6 +63,15 @@ def url_detail_view(request, id): } return render(request, 'url_detail.html', context) +def publish(request, id): + # Publish URL content to Ghost + try: + Publisher().publish(id) + message = "URL ID {} published".format(id) + except Exception as e: + message = "Error publishing URL ID {}: {}".format(id, str(e)) + return HttpResponse(message) + #################################################################################################### def charts(request): return render(request, 'charts.html') diff --git a/app_urls/init_db.py b/app_urls/init_db.py index 2d3950a..5e6dfdd 100644 --- a/app_urls/init_db.py +++ b/app_urls/init_db.py @@ -210,10 +210,9 @@ def initialize_data(): for list_pattern_status_priority in data_json.get("REGEX_PATTERN_STATUS_PRIORITY", []): # Decode pattern, status, priority = list_pattern_status_priority - # Query - query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES ('{}', {}, '{}');".format(pattern, priority, status) - print(query) - cur.execute(query) + # Query (parameterized to avoid SQL injection) + query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES (%s, %s, %s);" + cur.execute(query, (pattern, priority, status)) # Connect to an existing database with psycopg.connect(connection_info) as conn: @@ -222,9 +221,8 @@ def initialize_data(): # Feeds, URL host, keyword search for search_type, list_searches in data_json.get("SEARCH", {}).items(): for search in list_searches: - query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}');".format(search, search_type) - print(query) - cur.execute(query) + insert_search_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s);" + cur.execute(insert_search_sql, (search, search_type)) # Try finding RSS feed if (search_type == "url_host"): @@ -232,9 +230,8 @@ def initialize_data(): list_feeds = find_feeds(url_host) # If not exists, insert feed for feed in list_feeds: - query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}') ON CONFLICT DO NOTHING;".format(feed, "rss_feed") - print(query) - cur.execute(query) + insert_feed_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s) ON CONFLICT DO NOTHING;" + cur.execute(insert_feed_sql, (feed, "rss_feed")) if __name__ == '__main__': diff --git a/app_urls/scheduled_tasks.json b/app_urls/scheduled_tasks.json index af4c1bb..dcd7499 100644 --- a/app_urls/scheduled_tasks.json +++ b/app_urls/scheduled_tasks.json @@ -393,7 +393,7 @@ }, { "model": "django_celery_beat.periodictask", - "pk": 4, + "pk": 15, "fields": { "name": "Notify status", "task": "fetcher.tasks.notify_status", -- 2.54.0 From d18eda2fc4b89b51ca6ba1638babd148c3a1cb2b Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Wed, 2 Sep 2026 09:42:07 -0300 Subject: [PATCH 2/5] Add unit tests and Gitea CI workflow for bug fixes - Add fetcher/tests/ with 21 tests covering all bug fixes: - test_utils: get_with_protocol URL normalization (5 tests) - test_config: scheduled_tasks.json PK uniqueness, Celery queue settings (7 tests) - test_publish: URL routing and view for /task/publish_/ (4 tests) - test_template_field: url_detail.html renders images_url field (2 tests) - test_search_postprocess: URL host filtering and param stripping (3 tests) - Replace empty fetcher/tests.py with tests/ package - Add .gitea/workflows/test.yml: runs on PRs and pushes to main, installs dependencies and runs `manage.py test fetcher.tests` --- .gitea/workflows/test.yml | 57 ++++++++++++++++ app_urls/fetcher/tests.py | 3 - app_urls/fetcher/tests/__init__.py | 0 app_urls/fetcher/tests/test_config.py | 68 +++++++++++++++++++ app_urls/fetcher/tests/test_publish.py | 44 ++++++++++++ .../fetcher/tests/test_search_postprocess.py | 45 ++++++++++++ app_urls/fetcher/tests/test_template_field.py | 62 +++++++++++++++++ app_urls/fetcher/tests/test_utils.py | 22 ++++++ 8 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 .gitea/workflows/test.yml delete mode 100644 app_urls/fetcher/tests.py create mode 100644 app_urls/fetcher/tests/__init__.py create mode 100644 app_urls/fetcher/tests/test_config.py create mode 100644 app_urls/fetcher/tests/test_publish.py create mode 100644 app_urls/fetcher/tests/test_search_postprocess.py create mode 100644 app_urls/fetcher/tests/test_template_field.py create mode 100644 app_urls/fetcher/tests/test_utils.py diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 0000000..c19d3f8 --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,57 @@ +name: Tests + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: matitos + POSTGRES_USER: supermatitos + POSTGRES_PASSWORD: supermatitos + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + DJANGO_SECRET_KEY: ci-test-secret + DJANGO_ALLOWED_HOSTS: '*' + DJANGO_ALLOWED_ORIGINS: 'http://localhost' + DB_NAME: matitos + DB_USER: supermatitos + DB_PASSWORD: supermatitos + DB_HOST: localhost + DB_PORT: '5432' + REDIS_CACHE_HOST: localhost + REDIS_CACHE_PORT: '6379' + REDIS_CELERY_HOST: localhost + REDIS_CELERY_PORT: '6379' + PATH_LOGS_DIRECTORY: /tmp/logs + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: app_urls/requirements.txt + + - name: Install dependencies + run: pip install -r app_urls/requirements.txt + + - name: Run tests + working-directory: app_urls + run: python manage.py test fetcher.tests -v 2 diff --git a/app_urls/fetcher/tests.py b/app_urls/fetcher/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/app_urls/fetcher/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/app_urls/fetcher/tests/__init__.py b/app_urls/fetcher/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app_urls/fetcher/tests/test_config.py b/app_urls/fetcher/tests/test_config.py new file mode 100644 index 0000000..a1c329b --- /dev/null +++ b/app_urls/fetcher/tests/test_config.py @@ -0,0 +1,68 @@ +import json +import os + +from django.conf import settings +from django.test import SimpleTestCase + +SCHEDULED_TASKS_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "scheduled_tasks.json", +) + + +class ScheduledTasksFixtureTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + with open(SCHEDULED_TASKS_PATH) as f: + cls.data = json.load(f) + cls.periodic_tasks = [ + e + for e in cls.data + if e.get("model") == "django_celery_beat.periodictask" + ] + + def test_json_is_valid_and_loads(self): + self.assertIsInstance(self.data, list) + self.assertTrue(len(self.data) > 0) + + def test_periodic_task_primary_keys_are_unique(self): + pks = [e["pk"] for e in self.periodic_tasks] + self.assertEqual(len(pks), len(set(pks))) + + def test_notify_status_has_unique_pk(self): + notify = [ + e + for e in self.periodic_tasks + if e["fields"].get("task") == "fetcher.tasks.notify_status" + ] + self.assertEqual(len(notify), 1) + other_pks = { + e["pk"] + for e in self.periodic_tasks + if e["fields"].get("task") != "fetcher.tasks.notify_status" + } + self.assertNotIn(notify[0]["pk"], other_pks) + + def test_interval_schedule_primary_keys_are_unique(self): + schedules = [e for e in self.data if e["model"] == "django_celery_beat.intervalschedule"] + pks = [e["pk"] for e in schedules] + self.assertEqual(len(pks), len(set(pks))) + + def test_expected_critical_tasks_present(self): + tasks = {e["fields"].get("task") for e in self.periodic_tasks} + self.assertIn("fetcher.tasks.process_raw_urls", tasks) + self.assertIn("fetcher.tasks.fetch_feeds", tasks) + self.assertIn("fetcher.tasks.notify_status", tasks) + + +class CeleryQueueSettingsTests(SimpleTestCase): + def test_contains_worker_queues(self): + queues = [q.name for q in settings.CELERY_TASK_QUEUES] + self.assertIn("default", queues) + self.assertIn("light", queues) + self.assertIn("heavy", queues) + + def test_does_not_contain_deprecated_low_queue(self): + queues = [q.name for q in settings.CELERY_TASK_QUEUES] + self.assertNotIn("low", queues) diff --git a/app_urls/fetcher/tests/test_publish.py b/app_urls/fetcher/tests/test_publish.py new file mode 100644 index 0000000..91442a6 --- /dev/null +++ b/app_urls/fetcher/tests/test_publish.py @@ -0,0 +1,44 @@ +from django.test import RequestFactory, SimpleTestCase +from django.urls import resolve, reverse +from unittest.mock import patch + +from .. import views + + +class PublishRouteTests(SimpleTestCase): + def setUp(self): + self.factory = RequestFactory() + + def test_publish_url_resolves_to_publish_view(self): + match = resolve("/task/publish_42/") + self.assertEqual(match.func, views.publish) + self.assertEqual(match.kwargs, {"id": 42}) + + def test_publish_url_reverse(self): + url = reverse("publish", args=[7]) + self.assertEqual(url, "/task/publish_7/") + + @patch("fetcher.views.Publisher") + def test_publish_invokes_publisher(self, mock_publisher_cls): + mock_instance = mock_publisher_cls.return_value + mock_instance.publish.return_value = None + + request = self.factory.get("/task/publish_9/") + response = views.publish(request, 9) + + self.assertEqual(response.status_code, 200) + mock_publisher_cls.assert_called_once_with() + mock_instance.publish.assert_called_once_with(9) + self.assertIn(b"9", response.content) + + @patch("fetcher.views.Publisher") + def test_publish_reports_error_gracefully(self, mock_publisher_cls): + mock_instance = mock_publisher_cls.return_value + mock_instance.publish.side_effect = RuntimeError("boom") + + request = self.factory.get("/task/publish_3/") + response = views.publish(request, 3) + + self.assertEqual(response.status_code, 200) + self.assertIn(b"Error publishing", response.content) + mock_instance.publish.assert_called_once_with(3) diff --git a/app_urls/fetcher/tests/test_search_postprocess.py b/app_urls/fetcher/tests/test_search_postprocess.py new file mode 100644 index 0000000..4f0ef66 --- /dev/null +++ b/app_urls/fetcher/tests/test_search_postprocess.py @@ -0,0 +1,45 @@ +from django.test import SimpleTestCase + +from ..models import Search +from ..src.fetch_search_instances import FetcherAbstract + + +class _ConcreteFetcher(FetcherAbstract): + def _fetch_raw_urls(self, keyword_search): + raise NotImplementedError + + def _get_name(self): + return "test-fetcher" + + +class PostProcessUrlsTests(SimpleTestCase): + def setUp(self): + self.fetcher = _ConcreteFetcher() + + def test_url_host_filters_to_host(self): + obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST) + raw = [ + "https://www.cnbc.com/article/1", + "https://www.foxnews.com/article/2", + "cnbc.com/article/3", + ] + result = self.fetcher._post_process_urls(raw, obj_search) + self.assertEqual( + result, + ["https://www.cnbc.com/article/1", "cnbc.com/article/3"], + ) + + def test_keyword_search_keeps_all_urls(self): + obj_search = Search(search="child abuse", type=Search.TYPE_ENUM.KEYWORD_SEARCH) + raw = [ + "https://www.cnbc.com/article/1", + "https://www.foxnews.com/article/2", + ] + result = self.fetcher._post_process_urls(raw, obj_search) + self.assertEqual(result, raw) + + def test_query_parameters_are_removed(self): + obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST) + raw = ["https://www.cnbc.com/article/1?param=1234&h=yes"] + result = self.fetcher._post_process_urls(raw, obj_search) + self.assertEqual(result, ["https://www.cnbc.com/article/1"]) diff --git a/app_urls/fetcher/tests/test_template_field.py b/app_urls/fetcher/tests/test_template_field.py new file mode 100644 index 0000000..7ffc587 --- /dev/null +++ b/app_urls/fetcher/tests/test_template_field.py @@ -0,0 +1,62 @@ +from datetime import datetime + +from django.test import SimpleTestCase +from django.template.loader import get_template +from types import SimpleNamespace + + +def _build_url_item(): + return SimpleNamespace( + id=1, + url="https://example.com/article", + ts_fetch=datetime(2025, 7, 1, 12, 0, 0), + status="valid", + ) + + +def _build_url_content(images_url=None): + return SimpleNamespace( + date_published=datetime(2025, 7, 1, 11, 0, 0), + title="Example title", + description="Example description", + content="Example body", + valid_content=True, + language="en", + tags=["a", "b"], + authors=["Author"], + keywords=["kw"], + image_main_url="https://example.com/img.jpg", + images_url=images_url, + videos_url=["https://example.com/v.mp4"], + url_host="https://example.com", + site_name="Example", + ) + + +class UrlDetailTemplateTests(SimpleTestCase): + def _render(self, url_content): + template = get_template("url_detail.html") + context = { + "url_item": _build_url_item(), + "url_content": url_content, + "sources": ["feeds"], + "searches": ["child abuse"], + "models": ["llama3.2:3b"], + "prompt": "Summarize", + "url_canonical": None, + } + return template.render(context) + + def test_image_urls_field_renders(self): + html = self._render(_build_url_content(images_url=["https://example.com/1.jpg"])) + self.assertIn("https://example.com/1.jpg", html) + + def test_legacy_image_urls_attribute_is_not_referenced(self): + # Model exposes `images_url`, not `image_urls`. Rendering the template + # must not fail nor silently rely on a non-existent field. + with open( + "fetcher/templates/url_detail.html" + ) as f: + source = f.read() + self.assertNotIn("url_content.image_urls", source) + self.assertIn("url_content.images_url", source) diff --git a/app_urls/fetcher/tests/test_utils.py b/app_urls/fetcher/tests/test_utils.py new file mode 100644 index 0000000..cf26ea6 --- /dev/null +++ b/app_urls/fetcher/tests/test_utils.py @@ -0,0 +1,22 @@ +from django.test import SimpleTestCase +from ..src.utils import get_with_protocol + + +class GetWithProtocolTests(SimpleTestCase): + def test_missing_protocol_prepends_https(self): + self.assertEqual(get_with_protocol("example.com"), "https://example.com") + + def test_missing_protocol_with_path(self): + self.assertEqual( + get_with_protocol("example.com/article"), + "https://example.com/article", + ) + + def test_upgrades_http_to_https(self): + self.assertEqual(get_with_protocol("http://example.com"), "https://example.com") + + def test_preserves_https(self): + self.assertEqual(get_with_protocol("https://example.com"), "https://example.com") + + def test_empty_string_becomes_https(self): + self.assertEqual(get_with_protocol(""), "https://") -- 2.54.0 From fa9d552873a88494882a8beedb81c6d1cc895443 Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Wed, 2 Sep 2026 11:39:31 -0300 Subject: [PATCH 3/5] Point Search-Engines-Scraper dependency to Gitea mirror The Gitea CI runner has no outbound access to github.com. Mirror the repository on the local Gitea instance so pip install succeeds in CI. Co-Authored-By: opencode --- app_urls/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app_urls/requirements.txt b/app_urls/requirements.txt index a040de2..5f6c6a8 100644 --- a/app_urls/requirements.txt +++ b/app_urls/requirements.txt @@ -14,7 +14,7 @@ googlenewsdecoder gnews GoogleNews ddgs -git+https://github.com/tasos-py/Search-Engines-Scraper.git +git+https://gitea.matitos.org/matitos/search-engines-scraper.git furl langdetect ollama -- 2.54.0 From 9323afd68fd90fd3416058258b3b06afed82421c Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Wed, 2 Sep 2026 12:00:14 -0300 Subject: [PATCH 4/5] Make optional search imports lazy to avoid GitHub dependency in CI Wrap gnews, GoogleNews, and search_engines imports in try/except so the module loads without those packages installed. This allows the test suite to run on the Gitea runner without outbound GitHub access. Also revert requirements.txt back to the GitHub URL for the Search-Engines-Scraper dependency. Co-Authored-By: opencode --- .../fetcher/src/fetch_search_instances.py | 22 ++++++++++++++++--- app_urls/requirements.txt | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/app_urls/fetcher/src/fetch_search_instances.py b/app_urls/fetcher/src/fetch_search_instances.py index 586bd6c..3d82b7c 100644 --- a/app_urls/fetcher/src/fetch_search_instances.py +++ b/app_urls/fetcher/src/fetch_search_instances.py @@ -8,10 +8,26 @@ from .logger import get_logger logger = get_logger() from furl import furl -from gnews import GNews from ddgs import DDGS -from GoogleNews import GoogleNews -from search_engines import Yahoo, Aol + +try: + from gnews import GNews +except ImportError: + GNews = None + logger.warning("gnews not installed — SearchGNews disabled") + +try: + from GoogleNews import GoogleNews +except ImportError: + GoogleNews = None + logger.warning("GoogleNews not installed — SearchGoogleNews/SearchGoogleGeneral/SearchGoogleNewsRSS disabled") + +try: + from search_engines import Yahoo, Aol +except ImportError: + Yahoo = None + Aol = None + logger.warning("search_engines not installed — SearchYahooGeneral/SearchAOLGeneral disabled") ########################################################################### ########################################################################### diff --git a/app_urls/requirements.txt b/app_urls/requirements.txt index 5f6c6a8..a040de2 100644 --- a/app_urls/requirements.txt +++ b/app_urls/requirements.txt @@ -14,7 +14,7 @@ googlenewsdecoder gnews GoogleNews ddgs -git+https://gitea.matitos.org/matitos/search-engines-scraper.git +git+https://github.com/tasos-py/Search-Engines-Scraper.git furl langdetect ollama -- 2.54.0 From c7104ad8f0e4031282a1a6d1fc1bf83c1ba1bd78 Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Wed, 2 Sep 2026 12:06:37 -0300 Subject: [PATCH 5/5] Filter out git+https dependencies in CI The Gitea runner can't clone github.com repos via pip. Since the tests don't need Search-Engines-Scraper, exclude git+ lines from requirements during CI install. Co-Authored-By: opencode --- .gitea/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index c19d3f8..80f86c5 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -50,7 +50,7 @@ jobs: cache-dependency-path: app_urls/requirements.txt - name: Install dependencies - run: pip install -r app_urls/requirements.txt + run: grep -v '^git+' app_urls/requirements.txt | pip install -r /dev/stdin - name: Run tests working-directory: app_urls -- 2.54.0