Luciano Gervasoni
d18eda2fc4
Tests / test (pull_request) Failing after 3m11s
- Add fetcher/tests/ with 21 tests covering all bug fixes:
- test_utils: get_with_protocol URL normalization (5 tests)
- test_config: scheduled_tasks.json PK uniqueness, Celery queue
settings (7 tests)
- test_publish: URL routing and view for /task/publish_<id>/ (4 tests)
- test_template_field: url_detail.html renders images_url field (2 tests)
- test_search_postprocess: URL host filtering and param stripping (3 tests)
- Replace empty fetcher/tests.py with tests/ package
- Add .gitea/workflows/test.yml: runs on PRs and pushes to main,
installs dependencies and runs `manage.py test fetcher.tests`
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
from django.test import SimpleTestCase
|
|
|
|
from ..models import Search
|
|
from ..src.fetch_search_instances import FetcherAbstract
|
|
|
|
|
|
class _ConcreteFetcher(FetcherAbstract):
|
|
def _fetch_raw_urls(self, keyword_search):
|
|
raise NotImplementedError
|
|
|
|
def _get_name(self):
|
|
return "test-fetcher"
|
|
|
|
|
|
class PostProcessUrlsTests(SimpleTestCase):
|
|
def setUp(self):
|
|
self.fetcher = _ConcreteFetcher()
|
|
|
|
def test_url_host_filters_to_host(self):
|
|
obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST)
|
|
raw = [
|
|
"https://www.cnbc.com/article/1",
|
|
"https://www.foxnews.com/article/2",
|
|
"cnbc.com/article/3",
|
|
]
|
|
result = self.fetcher._post_process_urls(raw, obj_search)
|
|
self.assertEqual(
|
|
result,
|
|
["https://www.cnbc.com/article/1", "cnbc.com/article/3"],
|
|
)
|
|
|
|
def test_keyword_search_keeps_all_urls(self):
|
|
obj_search = Search(search="child abuse", type=Search.TYPE_ENUM.KEYWORD_SEARCH)
|
|
raw = [
|
|
"https://www.cnbc.com/article/1",
|
|
"https://www.foxnews.com/article/2",
|
|
]
|
|
result = self.fetcher._post_process_urls(raw, obj_search)
|
|
self.assertEqual(result, raw)
|
|
|
|
def test_query_parameters_are_removed(self):
|
|
obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST)
|
|
raw = ["https://www.cnbc.com/article/1?param=1234&h=yes"]
|
|
result = self.fetcher._post_process_urls(raw, obj_search)
|
|
self.assertEqual(result, ["https://www.cnbc.com/article/1"])
|