diff --git a/app_urls/core/settings.py b/app_urls/core/settings.py
index dd2effa..faa2dd6 100644
--- a/app_urls/core/settings.py
+++ b/app_urls/core/settings.py
@@ -123,7 +123,8 @@ CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler'
CELERY_TASK_QUEUES = (
Queue('default'),
- Queue('low'),
+ Queue('light'),
+ Queue('heavy'),
)
diff --git a/app_urls/fetcher/src/db_utils.py b/app_urls/fetcher/src/db_utils.py
index d0ba365..eac056e 100644
--- a/app_urls/fetcher/src/db_utils.py
+++ b/app_urls/fetcher/src/db_utils.py
@@ -212,7 +212,7 @@ class DB_Handler():
# Sort pattern tuples by priority. (pattern, priority, status)
for regex_pattern, regex_priority, status_if_match in sorted(list_pattern_status_tuple, key=lambda tup: tup[1], reverse=True):
# Regular expression pattern matching: https://regexr.com/
- if bool(re.match(regex_pattern, obj_url.url)):
+ if bool(re.match(regex_pattern, url)):
# logger.debug("Regex pattern found, status '{}' for URL: {}".format(status_if_match, url))
return status_if_match
return None
diff --git a/app_urls/fetcher/templates/url_detail.html b/app_urls/fetcher/templates/url_detail.html
index 5360209..3515647 100644
--- a/app_urls/fetcher/templates/url_detail.html
+++ b/app_urls/fetcher/templates/url_detail.html
@@ -246,7 +246,7 @@
| Image URLs |
- {{ url_content.image_urls|default:"" }} |
+ {{ url_content.images_url|default:"" }} |
| Video URLs |
diff --git a/app_urls/fetcher/urls.py b/app_urls/fetcher/urls.py
index 69605fb..a70434d 100644
--- a/app_urls/fetcher/urls.py
+++ b/app_urls/fetcher/urls.py
@@ -17,4 +17,5 @@ urlpatterns = [
#
path('urls/', views.filtered_urls, name='filtered_urls'),
path('urls//', views.url_detail_view, name='url_detail'),
+ path('task/publish_/', views.publish, name='publish'),
]
diff --git a/app_urls/fetcher/views.py b/app_urls/fetcher/views.py
index be15d61..39c96b0 100644
--- a/app_urls/fetcher/views.py
+++ b/app_urls/fetcher/views.py
@@ -2,12 +2,13 @@ from .views_base import link_list, logs, log_db #, trigger_task,
from django.core.paginator import Paginator
from django.shortcuts import render, get_object_or_404
-from django.http import StreamingHttpResponse, JsonResponse
+from django.http import StreamingHttpResponse, JsonResponse, HttpResponse
from django.db.models import Q, Count
from django.utils import timezone
from django.utils.timezone import now, timedelta
from .models import Urls, Source, Search, UrlContent, UrlsSourceSearch, UrlsDuplicate
from .src.llm import OllamaClient
+from .src.publisher import Publisher
import json
@@ -62,6 +63,15 @@ def url_detail_view(request, id):
}
return render(request, 'url_detail.html', context)
+def publish(request, id):
+ # Publish URL content to Ghost
+ try:
+ Publisher().publish(id)
+ message = "URL ID {} published".format(id)
+ except Exception as e:
+ message = "Error publishing URL ID {}: {}".format(id, str(e))
+ return HttpResponse(message)
+
####################################################################################################
def charts(request):
return render(request, 'charts.html')
diff --git a/app_urls/init_db.py b/app_urls/init_db.py
index 2d3950a..5e6dfdd 100644
--- a/app_urls/init_db.py
+++ b/app_urls/init_db.py
@@ -210,10 +210,9 @@ def initialize_data():
for list_pattern_status_priority in data_json.get("REGEX_PATTERN_STATUS_PRIORITY", []):
# Decode
pattern, status, priority = list_pattern_status_priority
- # Query
- query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES ('{}', {}, '{}');".format(pattern, priority, status)
- print(query)
- cur.execute(query)
+ # Query (parameterized to avoid SQL injection)
+ query = "INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES (%s, %s, %s);"
+ cur.execute(query, (pattern, priority, status))
# Connect to an existing database
with psycopg.connect(connection_info) as conn:
@@ -222,9 +221,8 @@ def initialize_data():
# Feeds, URL host, keyword search
for search_type, list_searches in data_json.get("SEARCH", {}).items():
for search in list_searches:
- query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}');".format(search, search_type)
- print(query)
- cur.execute(query)
+ insert_search_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s);"
+ cur.execute(insert_search_sql, (search, search_type))
# Try finding RSS feed
if (search_type == "url_host"):
@@ -232,9 +230,8 @@ def initialize_data():
list_feeds = find_feeds(url_host)
# If not exists, insert feed
for feed in list_feeds:
- query = "INSERT INTO SEARCH (search, type) VALUES ('{}', '{}') ON CONFLICT DO NOTHING;".format(feed, "rss_feed")
- print(query)
- cur.execute(query)
+ insert_feed_sql = "INSERT INTO SEARCH (search, type) VALUES (%s, %s) ON CONFLICT DO NOTHING;"
+ cur.execute(insert_feed_sql, (feed, "rss_feed"))
if __name__ == '__main__':
diff --git a/app_urls/scheduled_tasks.json b/app_urls/scheduled_tasks.json
index af4c1bb..dcd7499 100644
--- a/app_urls/scheduled_tasks.json
+++ b/app_urls/scheduled_tasks.json
@@ -393,7 +393,7 @@
},
{
"model": "django_celery_beat.periodictask",
- "pk": 4,
+ "pk": 15,
"fields": {
"name": "Notify status",
"task": "fetcher.tasks.notify_status",