From 8dce5206af7724b32716bc91db5f9113419defb7 Mon Sep 17 00:00:00 2001 From: Luciano Gervasoni Date: Wed, 26 Mar 2025 17:35:09 +0100 Subject: [PATCH] Better URL visualization --- app_urls/api/templates/filtered_urls.html | 256 ++++++++++++++++------ app_urls/api/urls.py | 5 +- app_urls/api/views.py | 55 +++-- 3 files changed, 237 insertions(+), 79 deletions(-) diff --git a/app_urls/api/templates/filtered_urls.html b/app_urls/api/templates/filtered_urls.html index 2be7028..9b50d2b 100644 --- a/app_urls/api/templates/filtered_urls.html +++ b/app_urls/api/templates/filtered_urls.html @@ -133,6 +133,7 @@ input[type="checkbox"] { + {% load custom_filters %}
@@ -140,8 +141,28 @@ input[type="checkbox"] {
+ +

Pages Per Page

+ +

+ + +

Fetch Date

+ +

+

Status

+
{% for status in statuses %}
-
- - +
+
+ + + + + + + + + + + + {% for url in urls %} - - - - - - - - - - {% for url in urls %} - - - - + + + + - - - - - {% empty %} - - - - {% endfor %} - -
IDURLStatusFetch DateSearchSource
IDURLStatusFetch DateSearchSource
{{ url.id }}{{ url.url }} - {% if url.status == 'raw' %} - {{ url.status|capfirst }} - {% elif url.status == 'error' %} - {{ url.status|capfirst }} - {% elif url.status == 'valid' %} - {{ url.status|capfirst }} - {% elif url.status == 'unknown' %} - {{ url.status|capfirst }} - {% elif url.status == 'invalid' %} - {{ url.status|capfirst }} - {% elif url.status == 'duplicate' %} - {{ url.status|capfirst }} + {{ url.id }}{{ url.url }} + {% if url.status == 'raw' %} + {{ url.status|capfirst }} + {% elif url.status == 'error' %} + {{ url.status|capfirst }} + {% elif url.status == 'valid' %} + {{ url.status|capfirst }} + {% elif url.status == 'unknown' %} + {{ url.status|capfirst }} + {% elif url.status == 'invalid' %} + {{ url.status|capfirst }} + {% elif url.status == 'duplicate' %} + {{ url.status|capfirst }} + {% else %} + Unknown + {% endif %} + + + + {% with sources_map|dict_get:url.id as sources %} + {% if sources %} + {% for source in sources %} + {{ source }} + {% endfor %} {% else %} - Unknown + No sources {% endif %} - {{ url.ts_fetch }} - {% for search in url.urlssourcesearch_set.all %} - {{ search.id_search.search }}
- {% endfor %} - 		- {% for source in url.urlssourcesearch_set.all %} - {{ source.id_source.source }}
- {% endfor %} -
No URLs found for the selected filters.
+ {% endwith %} + + + {% with searches_map|dict_get:url.id as searches %} + {% if searches %} + {% for search in searches %} + {{ search }} + {% endfor %} + {% else %} + No searches + {% endif %} + {% endwith %} + + + {% empty %} + + No URLs found for the selected filters. + + {% endfor %} + + + + + +
+ + diff --git a/app_urls/api/urls.py b/app_urls/api/urls.py index 34e839f..9da7bb9 100644 --- a/app_urls/api/urls.py +++ b/app_urls/api/urls.py @@ -13,10 +13,13 @@ urlpatterns = [ path('urls-per-source/', views.urls_per_source, name='urls_per_source'), path('urls-per-search/', views.urls_per_search, name='urls_per_search'), # - path('filtered-urls/', views.filtered_urls, name='filtered_urls'), + path('urls/', views.filtered_urls, name='filtered_urls'), + path('urls//', views.url_detail_view, name='url_detail'), + path('urls//fetch/', views.fetch_details, name='fetch_details'), # path('url/', views.urls, name='url_detail'), path('url//', views.url_detail_view, name='url_detail'), path('url//fetch/', views.fetch_details, name='fetch_details'), + # path('task/', views.trigger_task, name='trigger_task'), ] diff --git a/app_urls/api/views.py b/app_urls/api/views.py index 30b4411..0587559 100644 --- a/app_urls/api/views.py +++ b/app_urls/api/views.py @@ -109,7 +109,7 @@ class OllamaClient(): self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org")) def _get_default_model(self): - return "gemma3:1b" + return "llama3.2:3b" def get_models(self): models = sorted([m.model for m in self.client.list().models]) @@ -271,6 +271,8 @@ def logs(request): #################################################################################################### from django.shortcuts import render from .models import Urls, Search, Source +from django.db.models import Q +from django.utils.timezone import now, timedelta def filtered_urls(request): statuses = Urls.STATUS_ENUM.choices @@ -278,32 +280,51 @@ def filtered_urls(request): sources = Source.objects.all() # Check if filters are applied; if not, select all by default - if not request.GET: - selected_status = [str(status[0]) for status in statuses] - selected_search = [str(search.id) for search in searches] - selected_source = [str(source.id) for source in sources] - else: - selected_status = request.GET.getlist('status') - selected_search = request.GET.getlist('search') - selected_source = request.GET.getlist('source') + selected_status = request.GET.getlist('status', [str(status[0]) for status in statuses]) + selected_search = request.GET.getlist('search', [str(search.id) for search in searches]) + selected_source = request.GET.getlist('source', [str(source.id) for source in sources]) + selected_days = int(request.GET.get("selected_days", 30)) + + print(selected_days) # Filter URLs based on selected filters - urls = Urls.objects.all() - if selected_status: - urls = urls.filter(status__in=selected_status) - if selected_search: - urls = urls.filter(urlssourcesearch__id_search__in=selected_search) - if selected_source: - urls = urls.filter(urlssourcesearch__id_source__in=selected_source) + urls = Urls.objects.filter( + Q(urlssourcesearch__id_source__in=selected_source) & + Q(urlssourcesearch__id_search__in=selected_search) & + Q(status__in=selected_status) & + Q(ts_fetch__gte=now() - timedelta(days=selected_days)) + ).distinct() # .order_by('-ts_fetch') + + # Custom replace search type + for s in searches: + s.type = s.type.replace("rss_feed", "rss").replace("url_host", "url").replace("keyword_search", "keyword") + + # Pagination + per_page = request.GET.get('per_page', 25) # Default is 50 URLs per page + paginator = Paginator(urls, per_page) # Paginate the filtered URLs + page_number = request.GET.get('page') # Get the current page number + page_obj = paginator.get_page(page_number) # Get the current page object + + # Map URL IDs to their sources & searches, only for subset of URLs (page of interest) + sources_map = { + url.id: list(Source.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list + } + searches_map = { + url.id: list(Search.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list + } context = { - 'urls': urls, + 'urls': page_obj, # Pass the paginated URLs + 'per_page': per_page, # Send per_page value for dynamic pagination 'statuses': statuses, 'searches': searches, 'sources': sources, 'selected_status': selected_status, 'selected_search': selected_search, 'selected_source': selected_source, + "selected_days": selected_days, + "sources_map": sources_map, + "searches_map": searches_map, } return render(request, 'filtered_urls.html', context)