-
-
-
- | ID |
- URL |
- Status |
- Fetch Date |
- Search |
- Source |
-
-
-
- {% for url in urls %}
+
+
+
+
- | {{ url.id }} |
- {{ url.url }} |
-
- {% if url.status == 'raw' %}
- {{ url.status|capfirst }}
- {% elif url.status == 'error' %}
- {{ url.status|capfirst }}
- {% elif url.status == 'valid' %}
- {{ url.status|capfirst }}
- {% elif url.status == 'unknown' %}
- {{ url.status|capfirst }}
- {% elif url.status == 'invalid' %}
- {{ url.status|capfirst }}
- {% elif url.status == 'duplicate' %}
- {{ url.status|capfirst }}
- {% else %}
- Unknown
- {% endif %}
- |
-
-
- |
-
- {% with sources_map|dict_get:url.id as sources %}
- {% if sources %}
- {% for source in sources %}
- {{ source }}
- {% endfor %}
- {% else %}
- No sources
- {% endif %}
- {% endwith %}
- |
-
- {% with searches_map|dict_get:url.id as searches %}
- {% if searches %}
- {% for search in searches %}
- {{ search }}
- {% endfor %}
- {% else %}
- No searches
- {% endif %}
- {% endwith %}
- |
+ ID |
+ URL |
+ Status |
+ Fetch Date |
+ Search |
+ Source |
- {% empty %}
-
- | No URLs found for the selected filters. |
-
- {% endfor %}
-
-
+
+
+ {% for url in urls %}
+
+ | {{ url.id }} |
+ {{ url.url }} |
+
+ {% if url.status == 'raw' %}
+ {{ url.status|capfirst }}
+ {% elif url.status == 'error' %}
+ {{ url.status|capfirst }}
+ {% elif url.status == 'valid' %}
+ {{ url.status|capfirst }}
+ {% elif url.status == 'unknown' %}
+ {{ url.status|capfirst }}
+ {% elif url.status == 'invalid' %}
+ {{ url.status|capfirst }}
+ {% elif url.status == 'duplicate' %}
+ {{ url.status|capfirst }}
+ {% else %}
+ Unknown
+ {% endif %}
+ |
+
+
+ |
+
+ {% with sources_map|dict_get:url.id as sources %}
+ {% if sources %}
+ {% for source in sources %}
+ {{ source }}
+ {% endfor %}
+ {% else %}
+ No sources
+ {% endif %}
+ {% endwith %}
+ |
+
+ {% with searches_map|dict_get:url.id as searches %}
+ {% if searches %}
+ {% for search in searches %}
+ {{ search }}
+ {% endfor %}
+ {% else %}
+ No searches
+ {% endif %}
+ {% endwith %}
+ |
+
+ {% empty %}
+
+ | No URLs found for the selected filters. |
+
+ {% endfor %}
+
+
-
-
-
-
-
-
-
diff --git a/app_urls/api/urls.py b/app_urls/api/urls.py
index 9da7bb9..9822074 100644
--- a/app_urls/api/urls.py
+++ b/app_urls/api/urls.py
@@ -4,7 +4,8 @@ from . import views
urlpatterns = [
path('', views.link_list, name='link_list'),
#
- path('logs', views.logs, name='logs'),
+ path('logs_debug', views.logs_debug, name='logs_debug'),
+ path('logs_info', views.logs_info, name='logs_info'),
path('logs_error', views.logs_error, name='logs_error'),
#
path('charts/', views.charts, name='charts'),
diff --git a/app_urls/api/views.py b/app_urls/api/views.py
index e00b6ef..92a8ad7 100644
--- a/app_urls/api/views.py
+++ b/app_urls/api/views.py
@@ -28,7 +28,8 @@ def link_list(request):
# Charts
"http://localhost:8000/api/charts",
# Logs
- "http://localhost:8000/api/logs",
+ "http://localhost:8000/api/logs_debug",
+ "http://localhost:8000/api/logs_info",
"http://localhost:8000/api/logs_error",
# API tasks
] + [os.path.join(prefix, l) for l in links]
@@ -263,8 +264,13 @@ def logs_error(request):
file_content = f.read()
return HttpResponse(file_content, content_type="text/plain")
-def logs(request):
- with open(os.getenv("PATH_LOGS", "logs/log_app_fetcher.log"), "r") as f:
+def logs_info(request):
+ with open(os.getenv("PATH_LOGS_INFO", "logs/log_app_fetcher_info.log"), "r") as f:
+ file_content = f.read()
+ return HttpResponse(file_content, content_type="text/plain")
+
+def logs_debug(request):
+ with open(os.getenv("PATH_LOGS_DEBUG", "logs/log_app_fetcher_debug.log"), "r") as f:
file_content = f.read()
return HttpResponse(file_content, content_type="text/plain")
@@ -278,29 +284,40 @@ def filtered_urls(request):
statuses = Urls.STATUS_ENUM.choices
searches = Search.objects.all()
sources = Source.objects.all()
-
- # Check if filters are applied; if not, select all by default
- selected_status = request.GET.getlist('status', [str(status[0]) for status in statuses])
- selected_search = request.GET.getlist('search', [str(search.id) for search in searches])
- selected_source = request.GET.getlist('source', [str(source.id) for source in sources])
+
+ # Get selected parameters
+ selected_status = request.GET.getlist('status')
+ selected_search = request.GET.getlist('search')
+ selected_source = request.GET.getlist('source')
selected_days = request.GET.get("days", 30)
+ per_page = request.GET.get('per_page', 100) # Default is X URLs per page
+ page_number = request.GET.get('page') # Get the current page number
+
+ # charts = request.GET.get('charts', False)
+
+ # "Home" -> No parameters -> Override filter with default values
+ if ( len(request.GET.keys()) == 0 ):
+ selected_status = [str(status[0]) for status in statuses]
+ selected_search = [str(search.id) for search in searches]
+ selected_source = [str(source.id) for source in sources]
# Filter URLs based on selected filters
- urls = Urls.objects.filter(
- Q(urlssourcesearch__id_source__in=selected_source) &
- Q(urlssourcesearch__id_search__in=selected_search) &
- Q(status__in=selected_status) &
- Q(ts_fetch__gte=now() - timedelta(days=float(selected_days)))
- ).distinct() # .order_by('-ts_fetch')
+ if ('' in selected_status) or ('' in selected_search) or ('' in selected_source):
+ urls = []
+ else:
+ urls = Urls.objects.filter(
+ Q(urlssourcesearch__id_source__in=selected_source) &
+ Q(urlssourcesearch__id_search__in=selected_search) &
+ Q(status__in=selected_status) &
+ Q(ts_fetch__gte=now() - timedelta(days=float(selected_days)))
+ ).distinct() # .order_by('-ts_fetch')
# Custom replace search type
for s in searches:
s.type = s.type.replace("rss_feed", "rss").replace("url_host", "url").replace("keyword_search", "keyword")
# Pagination
- per_page = request.GET.get('per_page', 25) # Default is 50 URLs per page
paginator = Paginator(urls, per_page) # Paginate the filtered URLs
- page_number = request.GET.get('page') # Get the current page number
page_obj = paginator.get_page(page_number) # Get the current page object
# Map URL IDs to their sources & searches, only for subset of URLs (page of interest)
@@ -323,6 +340,7 @@ def filtered_urls(request):
"selected_days": selected_days,
"sources_map": sources_map,
"searches_map": searches_map,
+ # "charts": charts,
}
return render(request, 'filtered_urls.html', context)
diff --git a/app_urls/scheduled_tasks.json b/app_urls/scheduled_tasks.json
new file mode 100644
index 0000000..e62e585
--- /dev/null
+++ b/app_urls/scheduled_tasks.json
@@ -0,0 +1,128 @@
+[
+ {
+ "model": "RepeatableTaskType",
+ "name": "Fetch Feeds",
+ "callable": "api.tasks.fetch_feeds",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "default",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T14:33:56+00:00",
+ "interval": 15,
+ "interval_unit": "minutes",
+ "successful_runs": 215,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 14:18:58.028684+00:00",
+ "last_failed_run": null
+ },
+ {
+ "model": "RepeatableTaskType",
+ "name": "Process raw URLs",
+ "callable": "api.tasks.process_raw_urls",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "low",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T14:35:08+00:00",
+ "interval": 1,
+ "interval_unit": "hours",
+ "successful_runs": 41,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 13:35:48.534489+00:00",
+ "last_failed_run": null
+ },
+ {
+ "model": "RepeatableTaskType",
+ "name": "Process error URLs",
+ "callable": "api.tasks.process_error_urls",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "low",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T16:36:21+00:00",
+ "interval": 4,
+ "interval_unit": "hours",
+ "successful_runs": 10,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 12:37:28.301866+00:00",
+ "last_failed_run": null
+ },
+ {
+ "model": "RepeatableTaskType",
+ "name": "Fetch Parser",
+ "callable": "api.tasks.fetch_parser",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "default",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T14:25:42+00:00",
+ "interval": 1,
+ "interval_unit": "hours",
+ "successful_runs": 44,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 13:25:46.205433+00:00",
+ "last_failed_run": null
+ },
+ {
+ "model": "RepeatableTaskType",
+ "name": "Fetch Search",
+ "callable": "api.tasks.fetch_search",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "default",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T14:29:33+00:00",
+ "interval": 1,
+ "interval_unit": "hours",
+ "successful_runs": 46,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 13:33:00.628827+00:00",
+ "last_failed_run": null
+ },
+ {
+ "model": "RepeatableTaskType",
+ "name": "Process MissingKids URLs",
+ "callable": "api.tasks.process_missing_kids_urls",
+ "callable_args": [],
+ "callable_kwargs": [],
+ "enabled": true,
+ "queue": "default",
+ "repeat": null,
+ "at_front": false,
+ "timeout": null,
+ "result_ttl": 86400,
+ "cron_string": null,
+ "scheduled_time": "2025-03-27T14:37:50+00:00",
+ "interval": 2,
+ "interval_unit": "hours",
+ "successful_runs": 20,
+ "failed_runs": 0,
+ "last_successful_run": "2025-03-27 12:38:42.545373+00:00",
+ "last_failed_run": null
+ }
+]