Min num of sources filter, initialization scripts, docker ready to use dev mode

app_urls/fetcher/models.py

@@ -99,6 +99,7 @@ class UrlsDuplicate(models.Model):
 
 class UrlsSourceSearch(models.Model):
     id_url = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url', primary_key=True)  # The composite primary key (id_url, id_source, id_search) found, that is not supported. The first column is selected.
+    #id_url = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url')
     id_source = models.ForeignKey(Source, models.DO_NOTHING, db_column='id_source')
     id_search = models.ForeignKey(Search, models.DO_NOTHING, db_column='id_search')
 

app_urls/fetcher/templates/filtered_urls.html

@@ -331,6 +331,12 @@ input[type="checkbox"] {
                     </label><br>
                 {% endfor %}
 
+                <!-- Minimum Sources Count Box -->
+                <h3>Min #Sources</h3>
+                <div>
+                    <input type="number" id="minSourceCount" name="min_sources" value="{{ selected_min_sources }}" min="1" style="width: 60px; text-align: center;">
+                </div>
+
                 <!-- Filter by language -->
                 <h3>Language</h3>
                 <button type="button" class="toggle-all-btn" data-toggle="language">Toggle All</button><br>
@@ -538,6 +544,10 @@ input[type="checkbox"] {
                 }
             });
 
+            // Min number of sources
+            //const minSearchCount = document.getElementById('minSourceCount').value;
+            //params.set('min_search_count', minSearchCount);
+
             // Submit the form after updating all sections
             document.getElementById("filterForm").submit();
         }
@@ -566,6 +576,9 @@ input[type="checkbox"] {
                 updateFormParameters();
             });
         });
+        document.getElementById('minSourceCount').addEventListener('change', function() {
+            updateFormParameters();
+        });
         document.getElementById('perPageSelect').addEventListener('change', function() {
             updateFormParameters();
         });

app_urls/fetcher/templates/url_detail.html

@@ -198,7 +198,7 @@
             </tr>
             <tr>
                 <th>Status</th>
-                <td>{{ url_item.status }}</td>
+                <td>{{ url_item.status }} {% if url_canonical != None %}<a href="/urls/{{ url_canonical.id }}" target="_blank">[{{ url_canonical.id }}]</a>{% endif %} </td>
             </tr>
             <tr>
                 <th>URL host</th>

app_urls/fetcher/views.py

@@ -6,6 +6,8 @@ from django.contrib.auth.decorators import login_required
 import ollama
 from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch, UrlsDuplicate
 import os
+from .src.logger import get_logger
+logger = get_logger()
 
 ####################################################################################################
 def trigger_task(request, task):
@@ -94,13 +96,11 @@ def url_detail_view(request, id):
     url_item = get_object_or_404(Urls, id=id)
     url_sources = list(Source.objects.filter(urlssourcesearch__id_url=url_item).distinct())
     url_searches = list(Search.objects.filter(urlssourcesearch__id_url=url_item).distinct())
-    # url_source_search = UrlsSourceSearch.objects.filter(id_url=url_item)
-    
-    url_duplicate = UrlsDuplicate.objects.get(id_url_duplicated=url_item)
-    #id_url_canonical = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url_canonical', primary_key=True)  # The composite primary key (id_url_canonical, id_url_duplicated) found, that is not supported. The first column is selected.
-    #id_url_duplicated = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
-    
-    url_duplicate.id_url_duplicated
+
+    if (url_item.status == Urls.STATUS_ENUM.DUPLICATE):
+        url_canonical = UrlsDuplicate.objects.get(id_url_duplicated=url_item).id_url_canonical
+    else:
+        url_canonical = None
 
     try:
         url_content = UrlContent.objects.get(pk=id)
@@ -117,6 +117,7 @@ def url_detail_view(request, id):
         'models': ollama.get_models(),
         'prompt': ollama.get_prompt(),
         'url_content': url_content,
+        'url_canonical': url_canonical,
     }
     return render(request, 'url_detail.html', context)
 
@@ -232,6 +233,7 @@ def filtered_urls(request):
     selected_source = request.GET.getlist('source', ["null"])
     selected_language = request.GET.getlist('language', ["null"])
     selected_valid_contents = request.GET.getlist('valid_content', ["null"])
+    selected_min_sources = int(request.GET.get('min_sources', 1))
     selected_days = request.GET.get("days", 30)
     per_page = request.GET.get('per_page', 100)  # Default is X URLs per page
     page_number = request.GET.get('page')  # Get the current page number
@@ -298,6 +300,9 @@ def filtered_urls(request):
             # Update query
             query &= (subquery)
 
+        if (selected_min_sources > 1):
+            query &= Q(pk__in=UrlsSourceSearch.objects.values('id_url').annotate(search_count=Count('id_source', distinct=True)).filter(search_count__gte=selected_min_sources).values('id_url'))
+
         # Run query
         urls = Urls.objects.filter(query).distinct() # .order_by('-ts_fetch')
 
@@ -333,6 +338,7 @@ def filtered_urls(request):
         'selected_source': selected_source,
         'selected_language': selected_language,
         'selected_valid_contents': selected_valid_contents,
+        "selected_min_sources": selected_min_sources,
         "selected_days": selected_days,
         # Map
         "sources_map": sources_map,