URLs view refactor, article exception handling, visualize logs, charts

app_urls/api/views.py

@@ -25,6 +25,11 @@ def link_list(request):
         "http://localhost:8000/admin",
         # URLs
         "http://localhost:8000/api/url",
+        # Charts
+        "http://localhost:8000/api/charts",
+        # Logs
+        "http://localhost:8000/api/logs",
+        "http://localhost:8000/api/logs_error",
         # API tasks
     ] + [os.path.join(prefix, l) for l in links]
     # Json
@@ -98,7 +103,7 @@ def urls(request):
 
     return render(request, "urls.html", context)
 
-
+####################################################################################################
 class OllamaClient():
     def __init__(self):
         self.client = ollama.Client(host=os.getenv("ENDPOINT_OLLAMA", "https://ollamamodel.matitos.org"))
@@ -170,3 +175,137 @@ def fetch_details(request, id):
             yield chunk["message"]["content"]  # Stream each chunk of text
     
     return StreamingHttpResponse(stream_response(), content_type="text/plain")
+
+
+####################################################################################################
+from django.shortcuts import render
+from django.http import JsonResponse
+from django.db.models import Count
+from datetime import timedelta
+from django.utils import timezone
+from .models import Urls, UrlsSourceSearch
+
+def charts(request):
+    return render(request, 'charts.html')
+
+def urls_by_fetch_date(request):
+    # Get the date for 30 days ago
+    start_date = timezone.now() - timedelta(days=30)
+    
+    # Count the number of URLs grouped by fetch date
+    urls_data = Urls.objects.filter(ts_fetch__gte=start_date) \
+                            .values('ts_fetch__date') \
+                            .annotate(count=Count('id')) \
+                            .order_by('ts_fetch__date')
+    
+    # Format data to return as JSON
+    data = {
+        'dates': [item['ts_fetch__date'] for item in urls_data],
+        'counts': [item['count'] for item in urls_data],
+    }
+    
+    return JsonResponse(data)
+
+def urls_per_status(request):
+    # Get the filtering date parameter
+    days = int(request.GET.get('days', 30))  # Default is 30 days
+    start_date = timezone.now() - timedelta(days=days)
+
+    # Count the number of URLs grouped by status within the date range
+    urls_data = Urls.objects.filter(ts_fetch__gte=start_date) \
+                            .values('status') \
+                            .annotate(count=Count('id')) \
+                            .order_by('status')
+    
+    # Format data for JSON
+    data = {
+        'statuses': [item['status'] for item in urls_data],
+        'counts': [item['count'] for item in urls_data],
+    }
+    
+    return JsonResponse(data)
+
+def urls_per_source(request):
+    # Count the number of URLs grouped by source
+    urls_data = UrlsSourceSearch.objects \
+                                .values('id_source__source') \
+                                .annotate(count=Count('id_url')) \
+                                .order_by('id_source__source')
+    
+    # Format data for JSON
+    data = {
+        'sources': [item['id_source__source'] for item in urls_data],
+        'counts': [item['count'] for item in urls_data],
+    }
+    
+    return JsonResponse(data)
+
+def urls_per_search(request):
+    # Count the number of URLs grouped by search
+    urls_data = UrlsSourceSearch.objects \
+                                .values('id_search__search') \
+                                .annotate(count=Count('id_url')) \
+                                .order_by('id_search__search')
+    
+    # Format data for JSON
+    data = {
+        'searches': [item['id_search__search'] for item in urls_data],
+        'counts': [item['count'] for item in urls_data],
+    }
+    
+    return JsonResponse(data)
+
+####################################################################################################
+from django.http import HttpResponse
+
+def logs_error(request):
+    with open(os.getenv("PATH_LOGS_ERROR", "logs/log_app_fetcher_error.log"), "r") as f:
+        file_content = f.read()
+    return HttpResponse(file_content, content_type="text/plain")
+
+def logs(request):
+    with open(os.getenv("PATH_LOGS", "logs/log_app_fetcher.log"), "r") as f:
+        file_content = f.read()
+    return HttpResponse(file_content, content_type="text/plain")
+
+####################################################################################################
+from django.shortcuts import render
+from .models import Urls, Search, Source
+
+def filtered_urls(request):
+    statuses = Urls.STATUS_ENUM.choices
+    searches = Search.objects.all()
+    sources = Source.objects.all()
+
+    # Check if filters are applied; if not, select all by default
+    if not request.GET:
+        selected_status = [str(status[0]) for status in statuses]
+        selected_search = [str(search.id) for search in searches]
+        selected_source = [str(source.id) for source in sources]
+    else:
+        selected_status = request.GET.getlist('status')
+        selected_search = request.GET.getlist('search')
+        selected_source = request.GET.getlist('source')
+
+    # Filter URLs based on selected filters
+    urls = Urls.objects.all()
+    if selected_status:
+        urls = urls.filter(status__in=selected_status)
+    if selected_search:
+        urls = urls.filter(urlssourcesearch__id_search__in=selected_search)
+    if selected_source:
+        urls = urls.filter(urlssourcesearch__id_source__in=selected_source)
+
+    context = {
+        'urls': urls,
+        'statuses': statuses,
+        'searches': searches,
+        'sources': sources,
+        'selected_status': selected_status,
+        'selected_search': selected_search,
+        'selected_source': selected_source,
+    }
+
+    return render(request, 'filtered_urls.html', context)
+
+####################################################################################################