matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better URL visualization

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-03-26 17:35:09 +01:00
parent e1f4787119
commit 8dce5206af
3 changed files with 237 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
app_urls/api/templates/filtered_urls.html
View File

@@ -133,6 +133,7 @@ input[type="checkbox"] {
</style> </style>
</head> </head>
<body> <body>
{% load custom_filters %}
<div class="container"> <div class="container">
<div class="sidebar"> <div class="sidebar">
@@ -140,8 +141,28 @@ input[type="checkbox"] {
<form method="GET" action="" id="filterForm"> <form method="GET" action="" id="filterForm">
<!-- Pages Per Page Dropdown -->
<h3>Pages Per Page</h3>
<select id="perPageSelect" name="per_page">
<option value="25" {% if per_page == '25' %}selected{% endif %}>25</option>
<option value="100" {% if per_page == '100' %}selected{% endif %}>100</option>
<option value="500" {% if per_page == '500' %}selected{% endif %}>500</option>
</select>
<br><br>
<!-- Filter by Time Range -->
<h3>Fetch Date</h3>
<select id="timeFilterSelect" name="selected_days">
<option value="1" {% if selected_days|stringformat:"s" == '1' %}selected{% endif %}>Last 24 hours</option>
<option value="7" {% if selected_days|stringformat:"s" == '7' %}selected{% endif %}>Last 7 days</option>
<option value="30" {% if selected_days|stringformat:"s" == '30' %}selected{% endif %}>Last 30 days</option>
<option value="90" {% if selected_days|stringformat:"s" == '90' %}selected{% endif %}>Last 90 days</option>
</select>
<br><br>
<!-- Filter by Status --> <!-- Filter by Status -->
<h3>Status</h3> <h3>Status</h3>
<button type="button" class="toggle-all-btn" data-toggle="status">Toggle All</button><br>
{% for status in statuses %} {% for status in statuses %}
<label> <label>
<input type="checkbox" name="status" value="{{ status.0 }}" <input type="checkbox" name="status" value="{{ status.0 }}"
@@ -149,105 +170,137 @@ input[type="checkbox"] {
{{ status.1 }} {{ status.1 }}
</label><br> </label><br>
{% endfor %} {% endfor %}
<br><br>
<!-- Filter by Search --> <!-- Filter by Search -->
<h3>Search</h3> <h3>Search</h3>
<button type="button" class="toggle-all-btn" data-toggle="search">Toggle All</button><br>
{% for search in searches %} {% for search in searches %}
<label> <label>
<input type="checkbox" name="search" value="{{ search.id }}" <input type="checkbox" name="search" value="{{ search.id }}"
{% if search.id|stringformat:"s" in selected_search %}checked{% endif %}> {% if search.id|stringformat:"s" in selected_search %}checked{% endif %}>
[{{ search.type }}] {{ search.search }} [{{ search.type }}] {{ search.search|truncatechars:70 }}
</label><br> </label><br>
{% endfor %} {% endfor %}
<br><br>
<!-- Filter by Source --> <!-- Filter by Source -->
<h3>Source</h3> <h3>Source</h3>
<button type="button" class="toggle-all-btn" data-toggle="source">Toggle All</button><br>
{% for source in sources %} {% for source in sources %}
<label> <label>
<input type="checkbox" name="source" value="{{ source.id }}" <input type="checkbox" name="source" value="{{ source.id }}"
{% if source.id|stringformat:"s" in selected_source %}checked{% endif %}> {% if source.id|stringformat:"s" in selected_source %}checked{% endif %}>
{{ source.source }} {{ source.source|truncatechars:70 }}
</label><br> </label><br>
{% endfor %} {% endfor %}
<br><br>
</form> </form>
</div> </div>
<div class="table-container"> <div class="table-container">
<table> <table>
<thead> <thead>
<tr>
<th>ID</th>
<th>URL</th>
<th>Status</th>
<th>Fetch Date</th>
<th>Search</th>
<th>Source</th>
</tr>
</thead>
<tbody>
{% for url in urls %}
<tr> <tr>
<th>ID</th> <td><a href="./{{ url.id }}" class="btn btn-primary btn-sm" target="_blank">{{ url.id }}</a></td>
<th>URL</th> <td><a href="{{ url.url }}/" target="_blank">{{ url.url }}</a></td>
<th>Status</th> <td>
<th>Fetch Date</th> {% if url.status == 'raw' %}
<th>Search</th> <span class="badge bg-secondary">{{ url.status|capfirst }}</span>
<th>Source</th> {% elif url.status == 'error' %}
</tr> <span class="badge bg-danger">{{ url.status|capfirst }}</span>
</thead> {% elif url.status == 'valid' %}
<tbody> <span class="badge bg-success">{{ url.status|capfirst }}</span>
{% for url in urls %} {% elif url.status == 'unknown' %}
<tr> <span class="badge bg-warning">{{ url.status|capfirst }}</span>
<td><a href="./{{ url.id }}" class="btn btn-primary btn-sm" target="_blank">{{ url.id }}</a></td> {% elif url.status == 'invalid' %}
<td><a href="{{ url.url }}/" target="_blank">{{ url.url }}</a></td> <span class="badge bg-danger">{{ url.status|capfirst }}</span>
<td> {% elif url.status == 'duplicate' %}
{% if url.status == 'raw' %} <span class="badge bg-info">{{ url.status|capfirst }}</span>
<span class="badge bg-secondary">{{ url.status|capfirst }}</span> {% else %}
{% elif url.status == 'error' %} <span class="badge bg-light">Unknown</span>
<span class="badge bg-danger">{{ url.status|capfirst }}</span> {% endif %}
{% elif url.status == 'valid' %} </td>
<span class="badge bg-success">{{ url.status|capfirst }}</span> <td>
{% elif url.status == 'unknown' %} <span class="ts-fetch" data-ts="{{ url.ts_fetch|date:'c' }}"></span>
<span class="badge bg-warning">{{ url.status|capfirst }}</span> </td>
{% elif url.status == 'invalid' %} <td>
<span class="badge bg-danger">{{ url.status|capfirst }}</span> {% with sources_map|dict_get:url.id as sources %}
{% elif url.status == 'duplicate' %} {% if sources %}
<span class="badge bg-info">{{ url.status|capfirst }}</span> {% for source in sources %}
<span class="badge bg-secondary">{{ source }}</span>
{% endfor %}
{% else %} {% else %}
<span class="badge bg-light">Unknown</span> <span class="text-muted">No sources</span>
{% endif %} {% endif %}
</td> {% endwith %}
<td>{{ url.ts_fetch }}</td> </td>
<td> <td>
{% for search in url.urlssourcesearch_set.all %} {% with searches_map|dict_get:url.id as searches %}
{{ search.id_search.search }}<br> {% if searches %}
{% endfor %} {% for search in searches %}
</td> <span class="badge bg-secondary">{{ search }}</span>
<td> {% endfor %}
{% for source in url.urlssourcesearch_set.all %} {% else %}
{{ source.id_source.source }}<br> <span class="text-muted">No searches</span>
{% endfor %} {% endif %}
</td> {% endwith %}
</tr> </td>
{% empty %} </tr>
<tr> {% empty %}
<td colspan="5">No URLs found for the selected filters.</td> <tr>
</tr> <td colspan="5">No URLs found for the selected filters.</td>
{% endfor %} </tr>
</tbody> {% endfor %}
</table> </tbody>
</table>
<!-- Pagination Controls -->
<div class="pagination">
<div class="pagination-controls">
{% if urls.has_previous %}
<a href="#" class="pagination-link" data-page="1">« First</a>
<a href="#" class="pagination-link" data-page="{{ urls.previous_page_number }}">Previous</a>
{% endif %}
<span>Page {{ urls.number }} of {{ urls.paginator.num_pages }}</span>
{% if urls.has_next %}
<a href="#" class="pagination-link" data-page="{{ urls.next_page_number }}">Next</a>
<a href="#" class="pagination-link" data-page="{{ urls.paginator.num_pages }}">Last »</a>
{% endif %}
</div>
</div> </div>
</div> </div>
<!-- Passing the selected filters as JavaScript variables --> <!-- Passing the selected filters as JavaScript variables -->
<script type="text/javascript"> <script type="text/javascript">
// Make sure these variables are accessible in your JavaScript // Make sure these variables are accessible in your JavaScript
var selectedStatus = {{ selected_status|safe }}; var selectedStatus = {{ selected_status|safe }};
var selectedSearch = {{ selected_search|safe }}; var selectedSearch = {{ selected_search|safe }};
var selectedSource = {{ selected_source|safe }}; var selectedSource = {{ selected_source|safe }};
var perPage = {{ per_page|default:"25" }};
//var selectedDays = {{ selected_days|default:"30" }};
</script> </script>
<script> <script>
// Automatically submit the form when any checkbox changes //////////////////////////////////////////////////////////////////////
document.querySelectorAll('input[type="checkbox"]').forEach(function(checkbox) {
checkbox.addEventListener('change', function() {
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
});
});
document.addEventListener("DOMContentLoaded", function () { document.addEventListener("DOMContentLoaded", function () {
const themeToggle = document.getElementById("themeToggle"); const themeToggle = document.getElementById("themeToggle");
const body = document.body; const body = document.body;
@@ -270,7 +323,88 @@ input[type="checkbox"] {
themeToggle.textContent = "🌞"; themeToggle.textContent = "🌞";
} }
}); });
document.querySelectorAll(".ts-fetch").forEach(element => {
let utcDate = element.getAttribute("data-ts"); // Get timestamp from data attribute
let options = { year: 'numeric', month: 'numeric', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12:false};
if (utcDate) {
let localDate = new Date(utcDate).toLocaleString("en-GB", options); // Convert to local timezone
element.textContent = localDate; // Update the text content
}
});
}); });
//////////////////////////////////////////////////////////////////////
// Function to update pagination links
function updatePaginationLinks(pageNumber) {
// Get current URL and remove existing page parameter
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('page', pageNumber); // Update page parameter
window.location.href = currentUrl.toString(); // Redirect to the updated URL
}
// Attach event listeners to pagination links
document.querySelectorAll('.pagination-link').forEach(link => {
link.addEventListener('click', function(e) {
e.preventDefault();
const pageNumber = this.getAttribute('data-page');
updatePaginationLinks(pageNumber); // Update the page number in the URL
});
});
//////////////////////////////////////////////////////////////////////
// Function to toggle all checkboxes in a section
function toggleCheckboxes(section) {
const checkboxes = document.querySelectorAll(`[name='${section}']`);
const allChecked = Array.from(checkboxes).every(checkbox => checkbox.checked);
checkboxes.forEach(checkbox => {
checkbox.checked = !allChecked;
});
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
}
// Attach event listeners to "Toggle All" buttons
document.querySelectorAll('.toggle-all-btn').forEach(button => {
button.addEventListener('click', function() {
const section = this.getAttribute('data-toggle');
toggleCheckboxes(section);
});
});
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
// Automatically submit the form when any checkbox changes
document.querySelectorAll('input[type="checkbox"]').forEach(function(checkbox) {
checkbox.addEventListener('change', function() {
// Automatically submit the form when a checkbox is toggled
document.getElementById('filterForm').submit();
//const currentUrl = new URL(window.location.href);
//currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
//window.location.href = currentUrl.toString(); // Redirect to the updated URL with the new filter values
});
});
//////////////////////////////////////////////////////////////////////
// Automatically submit the form when per_page dropdown changes
document.getElementById('perPageSelect').addEventListener('change', function() {
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('per_page', this.value); // Update per_page value
currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
window.location.href = currentUrl.toString(); // Redirect to the updated URL with new per_page value
});
document.getElementById('timeFilterSelect').addEventListener('change', function() {
const currentUrl = new URL(window.location.href);
currentUrl.searchParams.set('selected_days', this.value); // Update per_page value
currentUrl.searchParams.set('page', 1); // Reset page number to 1 when any checkbox changes
window.location.href = currentUrl.toString(); // Redirect to the updated URL with new per_page value
});
</script> </script>
</body> </body>

5
app_urls/api/urls.py
View File

@@ -13,10 +13,13 @@ urlpatterns = [
path('urls-per-source/', views.urls_per_source, name='urls_per_source'), path('urls-per-source/', views.urls_per_source, name='urls_per_source'),
path('urls-per-search/', views.urls_per_search, name='urls_per_search'), path('urls-per-search/', views.urls_per_search, name='urls_per_search'),
# #
path('filtered-urls/', views.filtered_urls, name='filtered_urls'), path('urls/', views.filtered_urls, name='filtered_urls'),
path('urls/<int:id>/', views.url_detail_view, name='url_detail'),
path('urls/<int:id>/fetch/', views.fetch_details, name='fetch_details'),
# #
path('url/', views.urls, name='url_detail'), path('url/', views.urls, name='url_detail'),
path('url/<int:id>/', views.url_detail_view, name='url_detail'), path('url/<int:id>/', views.url_detail_view, name='url_detail'),
path('url/<int:id>/fetch/', views.fetch_details, name='fetch_details'), path('url/<int:id>/fetch/', views.fetch_details, name='fetch_details'),
#
path('task/<str:task>', views.trigger_task, name='trigger_task'), path('task/<str:task>', views.trigger_task, name='trigger_task'),
] ]

55
app_urls/api/views.py
View File

@@ -109,7 +109,7 @@ class OllamaClient():
self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org")) self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org"))
def _get_default_model(self): def _get_default_model(self):
return "gemma3:1b" return "llama3.2:3b"
def get_models(self): def get_models(self):
models = sorted([m.model for m in self.client.list().models]) models = sorted([m.model for m in self.client.list().models])
@@ -271,6 +271,8 @@ def logs(request):
#################################################################################################### ####################################################################################################
from django.shortcuts import render from django.shortcuts import render
from .models import Urls, Search, Source from .models import Urls, Search, Source
from django.db.models import Q
from django.utils.timezone import now, timedelta
def filtered_urls(request): def filtered_urls(request):
statuses = Urls.STATUS_ENUM.choices statuses = Urls.STATUS_ENUM.choices
@@ -278,32 +280,51 @@ def filtered_urls(request):
sources = Source.objects.all() sources = Source.objects.all()
# Check if filters are applied; if not, select all by default # Check if filters are applied; if not, select all by default
if not request.GET: selected_status = request.GET.getlist('status', [str(status[0]) for status in statuses])
selected_status = [str(status[0]) for status in statuses] selected_search = request.GET.getlist('search', [str(search.id) for search in searches])
selected_search = [str(search.id) for search in searches] selected_source = request.GET.getlist('source', [str(source.id) for source in sources])
selected_source = [str(source.id) for source in sources] selected_days = int(request.GET.get("selected_days", 30))
else:
selected_status = request.GET.getlist('status') print(selected_days)
selected_search = request.GET.getlist('search')
selected_source = request.GET.getlist('source')
# Filter URLs based on selected filters # Filter URLs based on selected filters
urls = Urls.objects.all() urls = Urls.objects.filter(
if selected_status: Q(urlssourcesearch__id_source__in=selected_source) &
urls = urls.filter(status__in=selected_status) Q(urlssourcesearch__id_search__in=selected_search) &
if selected_search: Q(status__in=selected_status) &
urls = urls.filter(urlssourcesearch__id_search__in=selected_search) Q(ts_fetch__gte=now() - timedelta(days=selected_days))
if selected_source: ).distinct() # .order_by('-ts_fetch')
urls = urls.filter(urlssourcesearch__id_source__in=selected_source)
# Custom replace search type
for s in searches:
s.type = s.type.replace("rss_feed", "rss").replace("url_host", "url").replace("keyword_search", "keyword")
# Pagination
per_page = request.GET.get('per_page', 25) # Default is 50 URLs per page
paginator = Paginator(urls, per_page) # Paginate the filtered URLs
page_number = request.GET.get('page') # Get the current page number
page_obj = paginator.get_page(page_number) # Get the current page object
# Map URL IDs to their sources & searches, only for subset of URLs (page of interest)
sources_map = {
url.id: list(Source.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
}
searches_map = {
url.id: list(Search.objects.filter(urlssourcesearch__id_url=url).distinct()) for url in page_obj.object_list
}
context = { context = {
'urls': urls, 'urls': page_obj, # Pass the paginated URLs
'per_page': per_page, # Send per_page value for dynamic pagination
'statuses': statuses, 'statuses': statuses,
'searches': searches, 'searches': searches,
'sources': sources, 'sources': sources,
'selected_status': selected_status, 'selected_status': selected_status,
'selected_search': selected_search, 'selected_search': selected_search,
'selected_source': selected_source, 'selected_source': selected_source,
"selected_days": selected_days,
"sources_map": sources_map,
"searches_map": searches_map,
} }
return render(request, 'filtered_urls.html', context) return render(request, 'filtered_urls.html', context)