Urls web visualization, cleaning obsolete code

2025-03-25 02:51:16 +01:00
parent 0c6b5f1ea4
commit 24b4614049
52 changed files with 371 additions and 3293 deletions
--- a/app_urls/api/models.py
+++ b/app_urls/api/models.py
@@ -17,7 +17,7 @@ class Search(models.Model):
        db_table = 'search'

    def __str__(self):
-        return "[{}]->{}".format(self.type, self.search)
+        return "[{}: {}]".format(self.type, self.search)

 class Source(models.Model):
    id = models.SmallAutoField(primary_key=True)
--- a/app_urls/api/src/db_utils.py
+++ b/app_urls/api/src/db_utils.py
@@ -130,7 +130,7 @@ class DB_Handler():
            # Get or create URL with canonical form
            obj_url_canonical, created = Urls.objects.get_or_create(url=dict_url_data.get("url_canonical"))
            # Get the source-search IDs associated to obj_url.id
-            list_url_source_search = UrlsSourceSearch.objects.fiter(id_url=obj_url)
+            list_url_source_search = UrlsSourceSearch.objects.filter(id_url=obj_url)
            for obj_url_source_search in list_url_source_search:
                # Associate same sources to url_canonical (it might already exist)
                UrlsSourceSearch.objects.get_or_create(id_url=obj_url_canonical, id_source=obj_url_source_search.id_source, id_search=obj_url_source_search.id_search)
--- a/app_urls/api/templates/item_list.html
+++ b/app_urls/api/templates/item_list.html
@@ -9,7 +9,7 @@

    <script>
        
-        function getQueryString(pageNumber, itemsNumber, sources, statuses){
+        function getQueryString(pageNumber, itemsNumber, sources, searches, statuses){
            // Query parameters. If input is null, get most recent value
            let queryParams = new URLSearchParams(window.location.search);
            // page
@@ -21,6 +21,9 @@
            // sources
            if (sources == null) sources = queryParams.get("sources") ?? "all";
            queryParams.set("sources", sources);
+            // searches
+            if (searches == null) searches = queryParams.get("searches") ?? "all";
+            queryParams.set("searches", searches);
            // status
            if (statuses == null) statuses = queryParams.get("status") ?? "all";
            queryParams.set("status", statuses);
@@ -33,11 +36,11 @@
            return queryParamsString;
        }

-        function loadPage(pageNumber, itemsNumber, sources, statuses) {
+        function loadPage(pageNumber, itemsNumber, sources, searches, statuses) {
            $("#item-list").fadeTo(100, 0.5); // Smooth fade effect
            $("#loading").show();
            
-            queryParamsString = getQueryString(pageNumber, itemsNumber, sources, statuses);
+            queryParamsString = getQueryString(pageNumber, itemsNumber, sources, searches, statuses);

            $.ajax({
                url: "?" + queryParamsString,
@@ -58,7 +61,7 @@
        $(document).on("click", ".pagination a", function (event) {
            event.preventDefault();
            let page = $(this).attr("data-page");
-            loadPage(pageNumber=page, itemsNumber=null, sources=null, statuses=null);
+            loadPage(pageNumber=page, itemsNumber=null, sources=null, searches=null, statuses=null);
        });

        $(document).ready(function () {
@@ -68,25 +71,63 @@
            ////////////////////////////////////////////////////////////////////////////
            const sourcesToggleAll = $("#toggle-all-sources");
            const sourcesCheckboxes = $(".source-checkbox");
+            const searchesToggleAll = $("#toggle-all-searches");
+            const searchesCheckboxes = $(".search-checkbox");
            const statusesToggleAll = $("#toggle-all-status");
            const statusCheckboxes = $(".status-checkbox");

            function updateFilters() {
-                // Get selected sources
-                let selectedSources = sourcesCheckboxes.filter(":checked").map(function () {
-                    return $(this).val();
-                }).get().join(",");
+                // Get selected sources                
+                if (sourcesToggleAll.prop("checked")) {
+                    selectedSources = "all";
+                }
+                else {
+                    if (sourcesCheckboxes.filter(":checked").length > 0 ){
+                        selectedSources = sourcesCheckboxes.filter(":checked").map(function () {
+                            return $(this).val();
+                        }).get().join(",");
+                    }
+                    else {
+                        selectedSources = "none";
+                    }
+                    
+                }
+
+                // Get selected searches
+                if (searchesToggleAll.prop("checked")) {
+                    selectedSearches = "all";
+                }
+                else {
+                    if (searchesCheckboxes.filter(":checked").length > 0 ){
+                        selectedSearches = searchesCheckboxes.filter(":checked").map(function () {
+                            return $(this).val();
+                        }).get().join(",");
+                    }
+                    else {
+                        selectedSearches = "none";
+                    }
+                }

                // Get selected URL statuses
-                let selectedStatuses = statusCheckboxes.filter(":checked").map(function () {
-                    return $(this).val();
-                }).get().join(",");
+                if (statusesToggleAll.prop("checked")) {
+                    selectedStatuses = "all";
+                }
+                else {
+                    if (statusCheckboxes.filter(":checked").length > 0 ){
+                        selectedStatuses = statusCheckboxes.filter(":checked").map(function () {
+                            return $(this).val();
+                        }).get().join(",");
+                    }
+                    else {
+                        selectedStatuses = "none";
+                    }
+                }

                // Get selected items per page
                let selectedItems = $("input[name='items']:checked").val();

                // Update pagination and reload data
-                loadPage(1, selectedItems, selectedSources, selectedStatuses);
+                loadPage(1, selectedItems, selectedSources, selectedSearches, selectedStatuses);
            }

            ////////////////////////////////////////////////////////////////////////////
@@ -101,6 +142,15 @@
                sourcesToggleAll.prop("checked", sourcesCheckboxes.length === sourcesCheckboxes.filter(":checked").length);
                updateFilters();
            });
+            // Searches
+            searchesToggleAll.on("change", function () {
+                searchesCheckboxes.prop("checked", searchesToggleAll.prop("checked"));
+                updateFilters();
+            });
+            searchesCheckboxes.on("change", function () {
+                searchesToggleAll.prop("checked", searchesCheckboxes.length === searchesCheckboxes.filter(":checked").length);
+                updateFilters();
+            });
            // Status
            statusesToggleAll.on("change", function () {
                statusCheckboxes.prop("checked", statusesToggleAll.prop("checked"));
@@ -121,11 +171,15 @@
            // Sources
            sourcesCheckboxes.each(function () { $(this).prop("checked", true); });
            sourcesToggleAll.prop("checked", true);
+            // Searches
+            searchesCheckboxes.each(function () { $(this).prop("checked", true); });
+            searchesToggleAll.prop("checked", true);
            // Statuses
            statusCheckboxes.each(function () { $(this).prop("checked", true); });
            statusesToggleAll.prop("checked", true);
            // Items
-            $("input[name='items'][value='" + 15 + "']").prop("checked", true);
+            // $("input[name='items'][value='" + 15 + "']").prop("checked", true);
+            // loadPage(pageNumber=page, itemsNumber=null, sources=null, searches=null, statuses=null);
        });

        ////////////////////////////////////////////////////////////////////////////
@@ -148,6 +202,23 @@
            let savedTheme = localStorage.getItem("theme") || 
                (window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light");
            setTheme(savedTheme);
+            // Local browser timestamp aware for ts_fetch print
+            document.querySelectorAll(".timestamp").forEach(function (el) {
+                const ts = el.getAttribute("data-ts");
+                if (ts) {
+                    const options = {
+                        day: "2-digit",
+                        month: "2-digit",
+                        year: "numeric",
+                        hour: "2-digit",
+                        minute: "2-digit",
+                        second: "2-digit",
+                        hour12: false  // Use 24-hour format
+                    }; // "en-GB" for DD-MM-YYYY
+                    const localDate = new Date(ts).toLocaleString("en-GB", options);  // Adjust to browser's timezone
+                    el.innerHTML = `${localDate}`;
+                }
+            });
        });
        ////////////////////////////////////////////////////////////////////////////
    </script>
@@ -174,6 +245,9 @@
            box-shadow: 2px 0 5px rgba(0, 0, 0, 0.1);
            padding: 15px;
            transition: width 0.3s ease;
+            /* Enable scrolling */
+            overflow-y: auto;
+            max-height: 100vh;
        }

        #sidebar .nav-link {
@@ -313,10 +387,10 @@
        }
        
        th:nth-child(1), td:nth-child(1) { width: 50%; } /* URL column */
-        th:nth-child(2), td:nth-child(2) { width: 20%; } /* Fetch Date */
-        th:nth-child(3), td:nth-child(3) { width: 20%; } /* Sources */
-        th:nth-child(4), td:nth-child(4) { width: 5%; } /* Status */
-        th:nth-child(5), td:nth-child(5) { width: 5%; } /* Action */
+        th:nth-child(2), td:nth-child(2) { width: 27.5%; } /* Fetch Date */
+        th:nth-child(3), td:nth-child(3) { width: 10%; } /* Sources */
+        th:nth-child(4), td:nth-child(4) { width: 10%; } /* Searches */
+        th:nth-child(5), td:nth-child(5) { width: 2.5%; } /* Status */

        /* ============================= */
        /* Pagination Styling            */
@@ -407,33 +481,23 @@
                    <span id="theme-icon">🌙</span>
                </button>
            </div>
-            
-            <!-- Sources -->
-            <div class="nav-item mt-3">
-                <strong>Select sources</strong>
-                <form id="source-filter-form">
-                    <!-- Toggle All Checkbox -->
-                    <div class="form-check">
-                        <input class="form-check-input" type="checkbox" id="toggle-all-sources">
-                        <label class="form-check-label fw-bold" for="toggle-all-sources">
-                            Toggle all
-                        </label>
-                    </div>

-                    <!-- Individual Source Checkboxes -->
-                    {% for source in sources %}
-                        <div class="form-check">
-                            <input class="form-check-input source-checkbox" type="checkbox" value="{{ source.id }}" id="source-{{ source.id }}">
-                            <label class="form-check-label" for="source-{{ source.id }}">
-                                {{ source.source }}
-                            </label>
+            <!-- URLs per page -->
+            <div class="nav-item mt-3">
+                <strong>URLs per page</strong>
+                <div class="card-body">
+                    <!-- Individual Status Checkboxes -->
+                    {% for url_per_page in list_urls_per_page %}
+                        <div class="items-form-check">
+                            <input class="form-check-input items" type="radio" name="items" id="value-{{ url_per_page }}" value="{{ url_per_page }}">
+                            <label class="form-check-label" for="value-{{ url_per_page }}">{{ url_per_page }}</label>
                        </div>
                    {% empty %}
                    <tr>
-                        <td colspan="2" class="text-center">No sources available.</td>
+                        <td colspan="2" class="text-center">No options available.</td>
                    </tr>
                    {% endfor %}
-                </form>
+                </div>
            </div>

            <!-- Status -->
@@ -457,6 +521,33 @@
                            </label>
                        </div>
                    {% empty %}
+                    <tr>
+                        <td colspan="2" class="text-center">No statuses available.</td>
+                    </tr>
+                    {% endfor %}
+                </form>
+            </div>
+            
+            <!-- Sources -->
+            <div class="nav-item mt-3">
+                <strong>Select sources</strong>
+                <form id="source-filter-form">
+                    <!-- Toggle All Checkbox -->
+                    <div class="form-check">
+                        <input class="form-check-input" type="checkbox" id="toggle-all-sources">
+                        <label class="form-check-label fw-bold" for="toggle-all-sources">
+                            Toggle all
+                        </label>
+                    </div>
+                    <!-- Individual Source Checkboxes -->
+                    {% for source in sources %}
+                        <div class="form-check">
+                            <input class="form-check-input source-checkbox" type="checkbox" value="{{ source.id }}" id="source-{{ source.id }}">
+                            <label class="form-check-label" for="source-{{ source.id }}">
+                                {{ source.source }}
+                            </label>
+                        </div>
+                    {% empty %}
                    <tr>
                        <td colspan="2" class="text-center">No sources available.</td>
                    </tr>
@@ -464,26 +555,34 @@
                </form>
            </div>

-            <!-- URLs per page -->
+            <!-- Searches -->
            <div class="nav-item mt-3">
-                <strong>URLs per page</strong>
-                <div class="card-body">
-                    <!-- Individual Status Checkboxes -->
-                    {% for url_per_page in list_urls_per_page %}
-                        <div class="items-form-check">
-                            <input class="form-check-input items" type="radio" name="items" id="value-{{ url_per_page }}" value="{{ url_per_page }}">
-                            <label class="form-check-label" for="value-{{ url_per_page }}">{{ url_per_page }}</label>
+                <strong>Select searches</strong>
+                <form id="search-filter-form">
+                    <!-- Toggle All Checkbox -->
+                    <div class="form-check">
+                        <input class="form-check-input" type="checkbox" id="toggle-all-searches">
+                        <label class="form-check-label fw-bold" for="toggle-all-searches">
+                            Toggle all
+                        </label>
+                    </div>
+                    <!-- Individual Search Checkboxes -->
+                    {% for search in searches %}
+                        <div class="form-check">
+                            <input class="form-check-input search-checkbox" type="checkbox" value="{{ search.id }}" id="search-{{ search.id }}">
+                            <label class="form-check-label" for="search-{{ search.id }}">
+                                [{{ search.type }}] {{ search.search }}
+                            </label>
                        </div>
                    {% empty %}
                    <tr>
-                        <td colspan="2" class="text-center">No options available.</td>
+                        <td colspan="2" class="text-center">No search available.</td>
                    </tr>
                    {% endfor %}
-                </div>
+                </form>
            </div>


-
        </ul>
    </div>

--- a/app_urls/api/templates/item_list_partial.html
+++ b/app_urls/api/templates/item_list_partial.html
@@ -7,15 +7,18 @@
                <th scope="col"><strong>URL</strong></th>
                <th scope="col"><strong>Fetch date</strong></th>
                <th scope="col"><strong>Sources</strong></th>
+                <th scope="col"><strong>Search</strong></th>
                <th scope="col"><strong>Status</strong></th>
-                <th scope="col"><strong>Action</strong></th>
            </tr>
        </thead>
        <tbody>
            {% for item in page_obj %}
                <tr>
-                    <td><a href="{{ item.url }}/" target="_blank">{{ item.url }}</a></td>
-                    <td>{{ item.ts_fetch }}</td>
+                    <td>
+                        <a href="./{{ item.id }}" class="btn btn-primary btn-sm" target="_blank">➤ </a>
+                        <a href="{{ item.url }}/" target="_blank">{{ item.url }}</a>
+                    </td>
+                    <td class="timestamp" data-ts="{{ item.ts_fetch|date:'c' }}">{{ item.ts_fetch }}</td>
                    <td>
                        {% with sources_map|dict_get:item.id as sources %}
                            {% if sources %}
@@ -27,6 +30,17 @@
                            {% endif %}
                        {% endwith %}
                    </td>
+                    <td>
+                        {% with searches_map|dict_get:item.id as searches %}
+                            {% if searches %}
+                                {% for search in searches %}
+                                    <span class="badge bg-secondary">{{ search }}</span>
+                                {% endfor %}
+                            {% else %}
+                                <span class="text-muted">No searches</span>
+                            {% endif %}
+                        {% endwith %}
+                    </td>
                    <td>
                        {% if item.status == 'raw' %}
                            <span class="badge bg-secondary">{{ item.status|capfirst }}</span>
@@ -43,11 +57,7 @@
                        {% else %}
                            <span class="badge bg-light">Unknown</span>
                        {% endif %}
-                    </td>
-                    <td>
-                        <a href="url/{{ item.id }}" class="btn btn-primary btn-sm" target="_blank">Details</a>  
-                    </td>
-                    
+                    </td>        
                </tr>
            {% empty %}
                <tr>
--- a/app_urls/api/templates/url_detail.html
+++ b/app_urls/api/templates/url_detail.html
@@ -54,7 +54,7 @@
        }

        // Fetch URL
-        let fetchUrl = `/news/url/${urlId}/fetch/?url=${encodeURIComponent(url)}&model=${encodeURIComponent(selectedModel)}&text=${encodeURIComponent(inputText)}`;
+        let fetchUrl = `/api/url/${urlId}/fetch/?url=${encodeURIComponent(url)}&model=${encodeURIComponent(selectedModel)}&text=${encodeURIComponent(inputText)}`;

        let resultContainer = $("#chat-output");
        resultContainer.html(""); // Clear previous content before fetching
@@ -99,12 +99,6 @@
                        // Render Markdown progressively (but safely)
                        messageContainer.html(marked.parse(accumulatedText));
                        //////////////////////////////////////
-                    
-                        //////////////////////////////////////
-                        // ORIGINAL:
-                        //let text = decoder.decode(value).replace(/\n/g, "<br>");
-                        //resultContainer.append(text); // Append streamed text
-                        //////////////////////////////////////

                        resultContainer.scrollTop(resultContainer[0].scrollHeight); // Auto-scroll to bottom
                        return read();
@@ -135,12 +129,16 @@
            </tr>
            <tr>
                <th>Fetch Date</th>
-                <td>{{ url_item.ts_fetch }}</td>
+                <td>{{ url_item.ts_fetch }} UTC</td>
            </tr>
            <tr>
-                <th>Sources</th>
+                <th>Source</th>
                <td>{{ sources|join:", " }}</td>
            </tr>
+            <tr>
+                <th>Search</th>
+                <td>{{ searches|join:", " }}</td>
+            </tr>
            <tr>
                <th>Status</th>
                <td>{{ url_item.status }}</td>
@@ -175,7 +173,6 @@
        <form onsubmit="fetchDetailsWithSelection(event, {{ url_item.id }}, '{{ url_item.url }}')">
            <label for="options-{{ url_item.id }}">Model:</label>
            <select id="options-{{ url_item.id }}" class="form-control mb-2">
-                <!-- <option value="">-- Select an option --</option> -->
                {% for model in models %}
                    <option value="{{ model }}">{{ model }}</option>
                {% endfor %}
@@ -185,21 +182,23 @@
        <!-- Input field with a default value -->        
        <label for="custom-input-{{ url_item.id }}">Prompt:</label>
        <textarea id="custom-input-{{ url_item.id }}" class="form-control mb-2" rows="3">{{ prompt }} {{ url_item.url }}</textarea>
-
-        <!-- Fetch details button -->
-        <button class="btn btn-primary" onclick="fetchDetails({{ url_item.id }}, '{{ url_item.url }}')">
-            Fetch Details
-        </button>
+        
+        <div class="d-flex align-items-center">
+            <!-- Fetch details button -->
+            <button class="btn btn-primary" onclick="fetchDetails({{ url_item.id }}, '{{ url_item.url }}')">
+                Fetch Details
+            </button>
+        
+            <!-- Loading Spinner (Hidden by Default) -->
+            <div id="loading-spinner" class="spinner-border text-primary ms-2" role="status" style="display: none;">
+                <span class="visually-hidden">Loading...</span>
+            </div>
+        </div>

        <!-- Chatbot-style response box -->
        <div class="chat-box mt-3 p-3 border rounded">
            <div id="chat-output"></div>
        </div>
-
-        <!-- Loading Spinner (Hidden by Default) -->
-        <div id="loading-spinner" class="spinner-border text-primary mt-3" role="status" style="display: none;">
-            <span class="visually-hidden">Loading...</span>
-        </div>
        
    </div>

--- a/app_urls/api/urls.py
+++ b/app_urls/api/urls.py
@@ -3,7 +3,7 @@ from . import views

 urlpatterns = [
    path('', views.link_list, name='link_list'),
-    path('url/', views.news, name='url_detail'),
+    path('url/', views.urls, name='url_detail'),
    path('url/<int:id>/', views.url_detail_view, name='url_detail'),
    path('url/<int:id>/fetch/', views.fetch_details, name='fetch_details'),
    path('task/<str:task>', views.trigger_task, name='trigger_task'),
--- a/app_urls/api/views.py
+++ b/app_urls/api/views.py
@@ -18,64 +18,80 @@ def link_list(request):
    prefix = "http://localhost:8000/api/task"
    links = ["fetch_feeds", "fetch_parser", "fetch_search", "process_raw_urls_50", "process_error_urls_50", "process_missing_kids_urls_50", "process_missing_kids_urls_500000"]

-    db_links = ["http://localhost:8080/?pgsql=matitos_db&username=supermatitos&db=matitos&ns=public&select=urls&order%5B0%5D=id&limit=500"]
-    return JsonResponse({"links": ["http://localhost:8000/api/url"] + db_links + [os.path.join(prefix, l) for l in links]})
+    list_links = [
+        # DB
+        "http://localhost:8080/?pgsql=matitos_db&username=supermatitos&db=matitos&ns=public&select=urls&order%5B0%5D=id&limit=500",
+        # Admin panel
+        "http://localhost:8000/admin",
+        # URLs
+        "http://localhost:8000/api/url",
+        # API tasks
+    ] + [os.path.join(prefix, l) for l in links]
+    # Json
+    return JsonResponse({"links": list_links })


-from django.http import StreamingHttpResponse, HttpResponse, JsonResponse
+from django.http import StreamingHttpResponse, JsonResponse
 from django.shortcuts import render, get_object_or_404
 from django.core.paginator import Paginator
-import requests
-from django.http import StreamingHttpResponse
-import json
-import time
 import ollama

-from .models import Urls, Source, Search, UrlsSourceSearch, UrlContent
+from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch

 # Create your views here.
-def news(request):
+def urls(request):
    # URLs
    urls = Urls.objects.all()
    # Sources
    sources = Source.objects.all()
-    seaerches = Search.objects.all()
+    searches = Search.objects.all()

    # Parameters
    page_number = request.GET.get("page", 1)
    num_items = request.GET.get("items", 15)
    source_ids = request.GET.get("sources", ','.join([str(s.id) for s in sources]))
+    search_ids = request.GET.get("searches", ','.join([str(s.id) for s in searches]))
    status_filters = request.GET.get("status", None)

    # Filters
    if (status_filters) and (status_filters != "all"):
-        urls = urls.filter(status__in=status_filters.split(","))
+        if (status_filters == "none"):
+            urls = []
+        else:
+            urls = urls.filter(status__in=status_filters.split(","))
    if (source_ids) and (source_ids != "all"):
-        # TODO: Distinct needed?
-        # urls = urls.filter(urlssource__id_source__in=source_ids.split(",")).distinct()
-        pass
+        if (source_ids == "none"):
+            urls = []
+        else:
+            urls = urls.filter(urlssourcesearch__id_source__in=source_ids.split(",")) # .distinct()
+    if (search_ids) and (search_ids != "all"):
+        if (search_ids == "none"):
+            urls = []
+        else:
+            urls = urls.filter(urlssourcesearch__id_search__in=search_ids.split(",")) # .distinct()

    # Pagination
    paginator = Paginator(urls, num_items)
    page_obj = paginator.get_page(page_number)

-    # Map URL IDs to their sources, only for subset of URLs (page of interest)
-    sources_map= {}
-    """
+    # Map URL IDs to their sources & searches, only for subset of URLs (page of interest)
    sources_map = {
-        url.id: list(Source.objects.filter(urlssource__id_url=url).values_list('source', flat=True))
-        for url in page_obj.object_list
+        url.id: list(Source.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
+    }
+    searches_map = {
+        url.id: list(Search.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
    }
-    """

    context = {
        "page_obj": page_obj,
        "sources": sources,
+        "searches": searches,
        "sources_map": sources_map,
+        "searches_map": searches_map,
        "list_status": Urls.STATUS_ENUM.values,
-        "list_urls_per_page": [15, 50, 100],
+        "list_urls_per_page": [15, 100, 500],
    }
-    
+
    # If request is AJAX, return JSON response
    if request.headers.get("X-Requested-With") == "XMLHttpRequest":
        return JsonResponse({'items_html': render(request, 'item_list_partial.html', context).content.decode('utf-8')})
@@ -83,32 +99,54 @@ def news(request):
    return render(request, "item_list.html", context)


+class OllamaClient():
+    def __init__(self):
+        self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org"))
+    
+    def _get_default_model(self):
+        return "gemma3:1b"
+
+    def get_models(self):
+        models = sorted([m.model for m in self.client.list().models])
+        if (self._get_default_model() in models):
+            return [self._get_default_model()] + [m for m in models if m != self._get_default_model()]
+        else:
+            return models
+    
+    def get_prompt(self):
+        return "Provide a summary of the content below, avoid mentioning the source of information, and only answer with the summary. The summary needs to be brief and compact, consisting of one paragraph."
+        #return "Explain in a single and compact paragraph the what, why, when, where, who, and how of the content below. Also provide a single paragraph summary of the content:"
+        #return "Provide in one paragraph the what, why, when, where, who, and how of the content below. Also provide a one paragraph summary of the content:"
+        #return "Provide two summaries of the content below, and avoid mentioning the source of information. First, provide a very brief and compact paragraph summary. Second, provide a larger and more detailed summary, which describe the what, why, when, where, who, and how of the content:"
+        # return "Imagine you are a journalist, TLDR in a paragraph. Only answer with the summary:"
+        #return "Below you will find the whole content of a news article:\n{}\nProvide a concise summary of one paragraph maximum of the content.".format(content)
+
+
 def url_detail_view(request, id):
    url_item = get_object_or_404(Urls, id=id)
-    url_sources = list(Source.objects.filter(urlssource__id_url=url_item).values_list('source', flat=True))
+    url_sources = list(Source.objects.filter(urlssourcesearch__id_url=url_item).distinct())
+    url_searches = list(Search.objects.filter(urlssourcesearch__id_url=url_item).distinct())
+    # url_source_search = UrlsSourceSearch.objects.filter(id_url=url_item)
+    
    try:
        url_content = UrlContent.objects.get(pk=id)
    except UrlContent.DoesNotExist:
        url_content = {}
    
    # TODO: https://github.com/ollama/ollama-python?tab=readme-ov-file#async-client
-    # LLM models available
-    client = ollama.Client(host = 'https://ollamamodel.matitos.org')
-    models = sorted([m.model for m in client.list().models])
-    # default_model = "llama3.2:3b"
+    ollama = OllamaClient()

    context = {
        'url_item': url_item,
        'sources': url_sources,
-        'models': models,
-        #'default_model': default_model,
-        'prompt': "Provide in one paragraph the what, why, when, where, who, and how of the content below. Also provide a one paragraph summary of the content:",
-        #"prompt": "Image you are a journalist, TLDR in a paragraph:",
-        #"prompt": "Below you will find the whole content of a news article:\n{}\nProvide a concise summary of one paragraph maximum of the content.".format(content)
+        'searches': url_searches,
+        'models': ollama.get_models(),
+        'prompt': ollama.get_prompt(),
        'url_content': url_content,
    }
    return render(request, 'url_detail.html', context)

+# TODO: move to ollamajs...
 def fetch_details(request, id):
    url_item = get_object_or_404(Urls, id=id)
    url_param = request.GET.get("url", "")  # Get URL
@@ -116,14 +154,14 @@ def fetch_details(request, id):
    text = request.GET.get("text", "")  # Get LLM prompt

    # LLM
-    client = ollama.Client(host = 'https://ollamamodel.matitos.org')
+    ollama = OllamaClient()

    def stream_response():
        msg_content = {
            "role": "user", 
            "content": text,
        }
-        response = client.chat(model=model, messages=[msg_content], stream=True)
+        response = ollama.client.chat(model=model, messages=[msg_content], stream=True)
        for chunk in response:
            yield chunk["message"]["content"]  # Stream each chunk of text
    
--- a/app_urls/core/settings.py
+++ b/app_urls/core/settings.py
@@ -124,9 +124,6 @@ SCHEDULER_QUEUES = {
        'PORT': os.environ.get("REDIS_PORT", 6379),
        'DB': os.environ.get("REDIS_DB", 0),
        'DEFAULT_TIMEOUT': os.environ.get("RQ_DEFAULT_TIMEOUT", 60*15),
-        #'USERNAME': 'some-user',
-        #'PASSWORD': 'some-password',
-        #'DEFAULT_TIMEOUT': 360,
  }
 }
 SCHEDULER_CONFIG = {
--- a/app_urls/core/urls.py
+++ b/app_urls/core/urls.py
@@ -20,6 +20,5 @@ from django.urls import path, include
 urlpatterns = [
    path('admin/', admin.site.urls),
    path('api/', include('api.urls')),
-    #path('scheduler/', include('django_rq.urls')),
    path('scheduler/', include('scheduler.urls')),
 ]