matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better URL visualization

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-03-26 17:35:09 +01:00
parent e1f4787119
commit 8dce5206af
3 changed files with 237 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
app_urls/api/templates/filtered_urls.html
View File

@@ -133,6 +133,7 @@ input[type="checkbox"] {
</style>
</head>
<body>
{% load custom_filters %}
<div class="container">
<div class="sidebar">
@@ -140,8 +141,28 @@ input[type="checkbox"] {
<form method="GET" action="" id="filterForm">
<!-- Pages Per Page Dropdown -->
<h3>Pages Per Page</h3>
<select id="perPageSelect" name="per_page">
<option value="25" {% if per_page == '25' %}selected{% endif %}>25</option>
<option value="100" {% if per_page == '100' %}selected{% endif %}>100</option>
<option value="500" {% if per_page == '500' %}selected{% endif %}>500</option>
</select>
<br><br>
<!-- Filter by Time Range -->
<h3>Fetch Date</h3>
<select id="timeFilterSelect" name="selected_days">
<option value="1" {% if selected_days|stringformat:"s" == '1' %}selected{% endif %}>Last 24 hours</option>
<option value="7" {% if selected_days|stringformat:"s" == '7' %}selected{% endif %}>Last 7 days</option>
<option value="30" {% if selected_days|stringformat:"s" == '30' %}selected{% endif %}>Last 30 days</option>
<option value="90" {% if selected_days|stringformat:"s" == '90' %}selected{% endif %}>Last 90 days</option>
</select>
<br><br>
<!-- Filter by Status -->
<h3>Status</h3>
<button type="button" class="toggle-all-btn" data-toggle="status">Toggle All</button><br>
{% for status in statuses %}
<label>
<input type="checkbox" name="status" value="{{ status.0 }}"
@@ -149,105 +170,137 @@ input[type="checkbox"] {
{{ status.1 }}
</label><br>
{% endfor %}
<br><br>
<!-- Filter by Search -->
<h3>Search</h3>
<button type="button" class="toggle-all-btn" data-toggle="search">Toggle All</button><br>
{% for search in searches %}
<label>
<input type="checkbox" name="search" value="{{ search.id }}"
{% if search.id|stringformat:"s" in selected_search %}checked{% endif %}>
[{{ search.type }}] {{ search.search }}
[{{ search.type }}] {{ search.search|truncatechars:70 }}
</label><br>
{% endfor %}
<br><br>
<!-- Filter by Source -->
<h3>Source</h3>
<button type="button" class="toggle-all-btn" data-toggle="source">Toggle All</button><br>
{% for source in sources %}
<label>
<input type="checkbox" name="source" value="{{ source.id }}"
{% if source.id|stringformat:"s" in selected_source %}checked{% endif %}>
{{ source.source }}
{{ source.source|truncatechars:70 }}
</label><br>
{% endfor %}
<br><br>
</form>
</div>
<div class="table-container">
<table>
<thead>
<div class="table-container">
<table>
<thead>
<tr>
<th>ID</th>
<th>URL</th>
<th>Status</th>
<th>Fetch Date</th>
<th>Search</th>
<th>Source</th>
</tr>
</thead>
<tbody>
{% for url in urls %}
<tr>
<th>ID</th>
<th>URL</th>
<th>Status</th>
<th>Fetch Date</th>
<th>Search</th>
<th>Source</th>
</tr>
</thead>
<tbody>
{% for url in urls %}
<tr>
<td><a href="./{{ url.id }}" class="btn btn-primary btn-sm" target="_blank">{{ url.id }}</a></td>
<td><a href="{{ url.url }}/" target="_blank">{{ url.url }}</a></td>
<td>
{% if url.status == 'raw' %}
<span class="badge bg-secondary">{{ url.status|capfirst }}</span>
{% elif url.status == 'error' %}
<span class="badge bg-danger">{{ url.status|capfirst }}</span>
{% elif url.status == 'valid' %}
<span class="badge bg-success">{{ url.status|capfirst }}</span>
{% elif url.status == 'unknown' %}
<span class="badge bg-warning">{{ url.status|capfirst }}</span>
{% elif url.status == 'invalid' %}
<span class="badge bg-danger">{{ url.status|capfirst }}</span>
{% elif url.status == 'duplicate' %}
<span class="badge bg-info">{{ url.status|capfirst }}</span>
<td><a href="./{{ url.id }}" class="btn btn-primary btn-sm" target="_blank">{{ url.id }}</a></td>
<td><a href="{{ url.url }}/" target="_blank">{{ url.url }}</a></td>
<td>
{% if url.status == 'raw' %}
<span class="badge bg-secondary">{{ url.status|capfirst }}</span>
{% elif url.status == 'error' %}
<span class="badge bg-danger">{{ url.status|capfirst }}</span>
{% elif url.status == 'valid' %}
<span class="badge bg-success">{{ url.status|capfirst }}</span>
{% elif url.status == 'unknown' %}
<span class="badge bg-warning">{{ url.status|capfirst }}</span>
{% elif url.status == 'invalid' %}
<span class="badge bg-danger">{{ url.status|capfirst }}</span>
{% elif url.status == 'duplicate' %}
<span class="badge bg-info">{{ url.status|capfirst }}</span>
{% else %}
<span class="badge bg-light">Unknown</span>
{% endif %}
</td>
<td>
<span class="ts-fetch" data-ts="{{ url.ts_fetch|date:'c' }}"></span>
</td>
<td>
{% with sources_map|dict_get:url.id as sources %}
{% if sources %}
{% for source in sources %}
<span class="badge bg-secondary">{{ source }}</span>
{% endfor %}
{% else %}
<span class="badge bg-light">Unknown</span>
<span class="text-muted">No sources</span>
{% endif %}
</td>
<td>{{ url.ts_fetch }}</td>
<td>
{% for search in url.urlssourcesearch_set.all %}
{{ search.id_search.search }}<br>
{% endfor %}
</td>
<td>
{% for source in url.urlssourcesearch_set.all %}
{{ source.id_source.source }}<br>
{% endfor %}
</td>
</tr>
{% empty %}
<tr>
<td colspan="5">No URLs found for the selected filters.</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endwith %}
</td>
<td>
{% with searches_map|dict_get:url.id as searches %}
{% if searches %}
{% for search in searches %}
<span class="badge bg-secondary">{{ search }}</span>
{% endfor %}
{% else %}
<span class="text-muted">No searches</span>
{% endif %}
{% endwith %}
</td>
</tr>
{% empty %}
<tr>
<td colspan="5">No URLs found for the selected filters.</td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- Pagination Controls -->
<div class="pagination">
<div class="pagination-controls">
{% if urls.has_previous %}
<a href="#" class="pagination-link" data-page="1">« First</a>
<a href="#" class="pagination-link" data-page="{{ urls.previous_page_number }}">Previous</a>
{% endif %}
<span>Page {{ urls.number }} of {{ urls.paginator.num_pages }}</span>
{% if urls.has_next %}
<a href="#" class="pagination-link" data-page="{{ urls.next_page_number }}">Next</a>
<a href="#" class="pagination-link" data-page="{{ urls.paginator.num_pages }}">Last »</a>
{% endif %}
</div>
</div>
</div>
<!-- Passing the selected filters as JavaScript variables -->
<script type="text/javascript">
// Make sure these variables are accessible in your JavaScript
var selectedStatus = {{ selected_status|safe }};
var selectedSearch = {{ selected_search|safe }};
var selectedSource = {{ selected_source|safe }};
var perPage = {{ per_page|default:"25" }};
//var selectedDays = {{ selected_days|default:"30" }};
</script>
<script>
// Automatically submit the form when any checkbox changes
document.querySelectorAll('input[type="checkbox"]').forEach(function(checkbox) {
checkbox.addEventListener('change', function() {
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
});
});
//////////////////////////////////////////////////////////////////////
document.addEventListener("DOMContentLoaded", function () {
const themeToggle = document.getElementById("themeToggle");
const body = document.body;
@@ -270,7 +323,88 @@ input[type="checkbox"] {
themeToggle.textContent = "🌞";
}
});
document.querySelectorAll(".ts-fetch").forEach(element => {
let utcDate = element.getAttribute("data-ts"); // Get timestamp from data attribute
let options = { year: 'numeric', month: 'numeric', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12:false};
if (utcDate) {
let localDate = new Date(utcDate).toLocaleString("en-GB", options); // Convert to local timezone
element.textContent = localDate; // Update the text content
}
});
});
//////////////////////////////////////////////////////////////////////
// Function to update pagination links
function updatePaginationLinks(pageNumber) {
// Get current URL and remove existing page parameter
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('page', pageNumber); // Update page parameter
window.location.href = currentUrl.toString(); // Redirect to the updated URL
}
// Attach event listeners to pagination links
document.querySelectorAll('.pagination-link').forEach(link => {
link.addEventListener('click', function(e) {
e.preventDefault();
const pageNumber = this.getAttribute('data-page');
updatePaginationLinks(pageNumber); // Update the page number in the URL
});
});
//////////////////////////////////////////////////////////////////////
// Function to toggle all checkboxes in a section
function toggleCheckboxes(section) {
const checkboxes = document.querySelectorAll(`[name='${section}']`);
const allChecked = Array.from(checkboxes).every(checkbox => checkbox.checked);
checkboxes.forEach(checkbox => {
checkbox.checked = !allChecked;
});
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
}
// Attach event listeners to "Toggle All" buttons
document.querySelectorAll('.toggle-all-btn').forEach(button => {
button.addEventListener('click', function() {
const section = this.getAttribute('data-toggle');
toggleCheckboxes(section);
});
});
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
// Automatically submit the form when any checkbox changes
document.querySelectorAll('input[type="checkbox"]').forEach(function(checkbox) {
checkbox.addEventListener('change', function() {
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
//const currentUrl = new URL(window.location.href);
//currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
//window.location.href = currentUrl.toString(); // Redirect to the updated URL with the new filter values
});
});
//////////////////////////////////////////////////////////////////////
// Automatically submit the form when per_page dropdown changes
document.getElementById('perPageSelect').addEventListener('change', function() {
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('per_page', this.value); // Update per_page value
currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
window.location.href = currentUrl.toString(); // Redirect to the updated URL with new per_page value
});
document.getElementById('timeFilterSelect').addEventListener('change', function() {
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('selected_days', this.value); // Update per_page value
currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
window.location.href = currentUrl.toString(); // Redirect to the updated URL with new per_page value
});
</script>
</body>

5
app_urls/api/urls.py
View File

@@ -13,10 +13,13 @@ urlpatterns = [
path('urls-per-source/', views.urls_per_source, name='urls_per_source'),
path('urls-per-search/', views.urls_per_search, name='urls_per_search'),
#
path('filtered-urls/', views.filtered_urls, name='filtered_urls'),
path('urls/', views.filtered_urls, name='filtered_urls'),
path('urls/<int:id>/', views.url_detail_view, name='url_detail'),
path('urls/<int:id>/fetch/', views.fetch_details, name='fetch_details'),
#
path('url/', views.urls, name='url_detail'),
path('url/<int:id>/', views.url_detail_view, name='url_detail'),
path('url/<int:id>/fetch/', views.fetch_details, name='fetch_details'),
#
path('task/<str:task>', views.trigger_task, name='trigger_task'),
]

55
app_urls/api/views.py
View File

@@ -109,7 +109,7 @@ class OllamaClient():
self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org"))
def _get_default_model(self):
return "gemma3:1b"
return "llama3.2:3b"
def get_models(self):
models = sorted([m.model for m in self.client.list().models])
@@ -271,6 +271,8 @@ def logs(request):
####################################################################################################
from django.shortcuts import render
from .models import Urls, Search, Source
from django.db.models import Q
from django.utils.timezone import now, timedelta
def filtered_urls(request):
statuses = Urls.STATUS_ENUM.choices
@@ -278,32 +280,51 @@ def filtered_urls(request):
sources = Source.objects.all()
# Check if filters are applied; if not, select all by default
if not request.GET:
selected_status = [str(status[0]) for status in statuses]
selected_search = [str(search.id) for search in searches]
selected_source = [str(source.id) for source in sources]
else:
selected_status = request.GET.getlist('status')
selected_search = request.GET.getlist('search')
selected_source = request.GET.getlist('source')
selected_status = request.GET.getlist('status', [str(status[0]) for status in statuses])
selected_search = request.GET.getlist('search', [str(search.id) for search in searches])
selected_source = request.GET.getlist('source', [str(source.id) for source in sources])
selected_days = int(request.GET.get("selected_days", 30))
print(selected_days)
# Filter URLs based on selected filters
urls = Urls.objects.all()
if selected_status:
urls = urls.filter(status__in=selected_status)
if selected_search:
urls = urls.filter(urlssourcesearch__id_search__in=selected_search)
if selected_source:
urls = urls.filter(urlssourcesearch__id_source__in=selected_source)
urls = Urls.objects.filter(
Q(urlssourcesearch__id_source__in=selected_source) &
Q(urlssourcesearch__id_search__in=selected_search) &
Q(status__in=selected_status) &
Q(ts_fetch__gte=now() - timedelta(days=selected_days))
).distinct() # .order_by('-ts_fetch')
# Custom replace search type
for s in searches:
s.type = s.type.replace("rss_feed", "rss").replace("url_host", "url").replace("keyword_search", "keyword")
# Pagination
per_page = request.GET.get('per_page', 25) # Default is 50 URLs per page
paginator = Paginator(urls, per_page) # Paginate the filtered URLs
page_number = request.GET.get('page') # Get the current page number
page_obj = paginator.get_page(page_number) # Get the current page object
# Map URL IDs to their sources & searches, only for subset of URLs (page of interest)
sources_map = {
url.id: list(Source.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
}
searches_map = {
url.id: list(Search.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
}
context = {
'urls': urls,
'urls': page_obj, # Pass the paginated URLs
'per_page': per_page, # Send per_page value for dynamic pagination
'statuses': statuses,
'searches': searches,
'sources': sources,
'selected_status': selected_status,
'selected_search': selected_search,
'selected_source': selected_source,
"selected_days": selected_days,
"sources_map": sources_map,
"searches_map": searches_map,
}
return render(request, 'filtered_urls.html', context)