diff --git a/app_urls/core/settings.py b/app_urls/core/settings.py index dd2effa..faa2dd6 100644 --- a/app_urls/core/settings.py +++ b/app_urls/core/settings.py @@ -123,7 +123,8 @@ CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' CELERY_TASK_QUEUES = ( Queue('default'), - Queue('low'), + Queue('light'), + Queue('heavy'), ) diff --git a/app_urls/fetcher/src/db_utils.py b/app_urls/fetcher/src/db_utils.py index d0ba365..eac056e 100644 --- a/app_urls/fetcher/src/db_utils.py +++ b/app_urls/fetcher/src/db_utils.py @@ -212,7 +212,7 @@ class DB_Handler(): # Sort pattern tuples by priority. (pattern, priority, status) for regex_pattern, regex_priority, status_if_match in sorted(list_pattern_status_tuple, key=lambda tup: tup[1], reverse=True): # Regular expression pattern matching: https://regexr.com/ - if bool(re.match(regex_pattern, obj_url.url)): + if bool(re.match(regex_pattern, url)): # logger.debug("Regex pattern found, status '{}' for URL: {}".format(status_if_match, url)) return status_if_match return None diff --git a/app_urls/fetcher/templates/url_detail.html b/app_urls/fetcher/templates/url_detail.html index 5360209..3515647 100644 --- a/app_urls/fetcher/templates/url_detail.html +++ b/app_urls/fetcher/templates/url_detail.html @@ -246,7 +246,7 @@ Image URLs - {{ url_content.image_urls|default:"" }} + {{ url_content.images_url|default:"" }} Video URLs diff --git a/app_urls/fetcher/urls.py b/app_urls/fetcher/urls.py index 69605fb..a70434d 100644 --- a/app_urls/fetcher/urls.py +++ b/app_urls/fetcher/urls.py @@ -17,4 +17,5 @@ urlpatterns = [ # path('urls/', views.filtered_urls, name='filtered_urls'), path('urls//', views.url_detail_view, name='url_detail'), + path('task/publish_/', views.publish, name='publish'), ] diff --git a/app_urls/fetcher/views.py b/app_urls/fetcher/views.py index be15d61..39c96b0 100644 --- a/app_urls/fetcher/views.py +++ b/app_urls/fetcher/views.py @@ -2,12 +2,13 @@ from .views_base import link_list, logs, log_db #, trigger_task, from django.core.paginator import Paginator from django.shortcuts import render, get_object_or_404 -from django.http import StreamingHttpResponse, JsonResponse +from django.http import StreamingHttpResponse, JsonResponse, HttpResponse from django.db.models import Q, Count from django.utils import timezone from django.utils.timezone import now, timedelta from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch, UrlsDuplicate from .src.llm import OllamaClient +from .src.publisher import Publisher import json @@ -62,6 +63,15 @@ def url_detail_view(request, id): } return render(request, 'url_detail.html', context) +def publish(request, id): + # Publish URL content to Ghost + try: + Publisher().publish(id) + message = "URL ID {} published".format(id) + except Exception as e: + message = "Error publishing URL ID {}: {}".format(id, str(e)) + return HttpResponse(message) + #################################################################################################### def charts(request): return render(request, 'charts.html') diff --git a/app_urls/init_db.py b/app_urls/init_db.py index 2d3950a..5e6dfdd 100644 --- a/app_urls/init_db.py +++ b/app_urls/init_db.py @@ -210,10 +210,9 @@ def initialize_data(): for list_pattern_status_priority in data_json.get("REGEX_PATTERN_STATUS_PRIORITY", []): # Decode pattern, status, priority = list_pattern_status_priority - # Query - query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES ('{}', {}, '{}');".format(pattern, priority, status) - print(query) - cur.execute(query) + # Query (parameterized to avoid SQL injection) + query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES (%s, %s, %s);" + cur.execute(query, (pattern, priority, status)) # Connect to an existing database with psycopg.connect(connection_info) as conn: @@ -222,9 +221,8 @@ def initialize_data(): # Feeds, URL host, keyword search for search_type, list_searches in data_json.get("SEARCH", {}).items(): for search in list_searches: - query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}');".format(search, search_type) - print(query) - cur.execute(query) + insert_search_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s);" + cur.execute(insert_search_sql, (search, search_type)) # Try finding RSS feed if (search_type == "url_host"): @@ -232,9 +230,8 @@ def initialize_data(): list_feeds = find_feeds(url_host) # If not exists, insert feed for feed in list_feeds: - query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}') ON CONFLICT DO NOTHING;".format(feed, "rss_feed") - print(query) - cur.execute(query) + insert_feed_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s) ON CONFLICT DO NOTHING;" + cur.execute(insert_feed_sql, (feed, "rss_feed")) if __name__ == '__main__': diff --git a/app_urls/scheduled_tasks.json b/app_urls/scheduled_tasks.json index af4c1bb..dcd7499 100644 --- a/app_urls/scheduled_tasks.json +++ b/app_urls/scheduled_tasks.json @@ -393,7 +393,7 @@ }, { "model": "django_celery_beat.periodictask", - "pk": 4, + "pk": 15, "fields": { "name": "Notify status", "task": "fetcher.tasks.notify_status",