- Add fetcher/tests/ with 21 tests covering all bug fixes:
- test_utils: get_with_protocol URL normalization (5 tests)
- test_config: scheduled_tasks.json PK uniqueness, Celery queue
settings (7 tests)
- test_publish: URL routing and view for /task/publish_<id>/ (4 tests)
- test_template_field: url_detail.html renders images_url field (2 tests)
- test_search_postprocess: URL host filtering and param stripping (3 tests)
- Replace empty fetcher/tests.py with tests/ package
- Add .gitea/workflows/test.yml: runs on PRs and pushes to main,
installs dependencies and runs `manage.py test fetcher.tests`
This commit is contained in:
1 parent
3a33697ca3
commit
d18eda2fc4
8 files changed
+298
-3
No files matched your search
@@ -0,0 +1,45 @@
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
from ..models import Search
|
||||
from ..src.fetch_search_instances import FetcherAbstract
|
||||
|
||||
|
||||
class _ConcreteFetcher(FetcherAbstract):
|
||||
def _fetch_raw_urls(self, keyword_search):
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_name(self):
|
||||
return "test-fetcher"
|
||||
|
||||
|
||||
class PostProcessUrlsTests(SimpleTestCase):
|
||||
def setUp(self):
|
||||
self.fetcher = _ConcreteFetcher()
|
||||
|
||||
def test_url_host_filters_to_host(self):
|
||||
obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST)
|
||||
raw = [
|
||||
"https://www.cnbc.com/article/1",
|
||||
"https://www.foxnews.com/article/2",
|
||||
"cnbc.com/article/3",
|
||||
]
|
||||
result = self.fetcher._post_process_urls(raw, obj_search)
|
||||
self.assertEqual(
|
||||
result,
|
||||
["https://www.cnbc.com/article/1", "cnbc.com/article/3"],
|
||||
)
|
||||
|
||||
def test_keyword_search_keeps_all_urls(self):
|
||||
obj_search = Search(search="child abuse", type=Search.TYPE_ENUM.KEYWORD_SEARCH)
|
||||
raw = [
|
||||
"https://www.cnbc.com/article/1",
|
||||
"https://www.foxnews.com/article/2",
|
||||
]
|
||||
result = self.fetcher._post_process_urls(raw, obj_search)
|
||||
self.assertEqual(result, raw)
|
||||
|
||||
def test_query_parameters_are_removed(self):
|
||||
obj_search = Search(search="cnbc.com", type=Search.TYPE_ENUM.URL_HOST)
|
||||
raw = ["https://www.cnbc.com/article/1?param=1234&h=yes"]
|
||||
result = self.fetcher._post_process_urls(raw, obj_search)
|
||||
self.assertEqual(result, ["https://www.cnbc.com/article/1"])
|
||||
Reference in new issue
Block a user