Min num of sources filter, initialization scripts, docker ready to use dev mode

app_urls/fetcher/views.py

@@ -6,6 +6,8 @@ from django.contrib.auth.decorators import login_required
 import ollama
 from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch, UrlsDuplicate
 import os
+from .src.logger import get_logger
+logger = get_logger()
 
 ####################################################################################################
 def trigger_task(request, task):
@@ -94,13 +96,11 @@ def url_detail_view(request, id):
     url_item = get_object_or_404(Urls, id=id)
     url_sources = list(Source.objects.filter(urlssourcesearch__id_url=url_item).distinct())
     url_searches = list(Search.objects.filter(urlssourcesearch__id_url=url_item).distinct())
-    # url_source_search = UrlsSourceSearch.objects.filter(id_url=url_item)
-    
-    url_duplicate = UrlsDuplicate.objects.get(id_url_duplicated=url_item)
-    #id_url_canonical = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url_canonical', primary_key=True)  # The composite primary key (id_url_canonical, id_url_duplicated) found, that is not supported. The first column is selected.
-    #id_url_duplicated = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
-    
-    url_duplicate.id_url_duplicated
+
+    if (url_item.status == Urls.STATUS_ENUM.DUPLICATE):
+        url_canonical = UrlsDuplicate.objects.get(id_url_duplicated=url_item).id_url_canonical
+    else:
+        url_canonical = None
 
     try:
         url_content = UrlContent.objects.get(pk=id)
@@ -117,6 +117,7 @@ def url_detail_view(request, id):
         'models': ollama.get_models(),
         'prompt': ollama.get_prompt(),
         'url_content': url_content,
+        'url_canonical': url_canonical,
     }
     return render(request, 'url_detail.html', context)
 
@@ -232,6 +233,7 @@ def filtered_urls(request):
     selected_source = request.GET.getlist('source', ["null"])
     selected_language = request.GET.getlist('language', ["null"])
     selected_valid_contents = request.GET.getlist('valid_content', ["null"])
+    selected_min_sources = int(request.GET.get('min_sources', 1))
     selected_days = request.GET.get("days", 30)
     per_page = request.GET.get('per_page', 100)  # Default is X URLs per page
     page_number = request.GET.get('page')  # Get the current page number
@@ -298,6 +300,9 @@ def filtered_urls(request):
             # Update query
             query &= (subquery)
 
+        if (selected_min_sources > 1):
+            query &= Q(pk__in=UrlsSourceSearch.objects.values('id_url').annotate(search_count=Count('id_source', distinct=True)).filter(search_count__gte=selected_min_sources).values('id_url'))
+
         # Run query
         urls = Urls.objects.filter(query).distinct() # .order_by('-ts_fetch')
 
@@ -333,6 +338,7 @@ def filtered_urls(request):
         'selected_source': selected_source,
         'selected_language': selected_language,
         'selected_valid_contents': selected_valid_contents,
+        "selected_min_sources": selected_min_sources,
         "selected_days": selected_days,
         # Map
         "sources_map": sources_map,