diff --git a/1-DB.ipynb b/1-DB.ipynb
new file mode 100644
index 0000000..bc72ed1
--- /dev/null
+++ b/1-DB.ipynb
@@ -0,0 +1,197 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install psycopg[binary]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "INSERT_TABLES = False\n",
+ "INSERT_SAMPLE_DATA = False\n",
+ "\n",
+ "import psycopg\n",
+ "connection_info = \"host={} port={} user={} password={} dbname={}\".format(\"localhost\", \"5432\", \"supermatitos\", \"supermatitos\", \"matitos\")\n",
+ "\n",
+ "\n",
+ "if INSERT_TABLES:\n",
+ " # Connect to an existing database\n",
+ " with psycopg.connect(connection_info) as conn:\n",
+ " # Open a cursor to perform database operations\n",
+ " with conn.cursor() as cur:\n",
+ " # Autocommit at end of transaction (Atomic insert of URLs and sources)\n",
+ " with conn.transaction() as tx:\n",
+ " # Create URLs table\n",
+ " c = cur.execute(\"\"\"\n",
+ " CREATE TYPE URL_STATUS AS ENUM ('raw', 'error', 'valid', 'unknown', 'invalid', 'duplicate');\n",
+ "\n",
+ " CREATE TABLE URLS (\n",
+ " id SERIAL PRIMARY KEY,\n",
+ " url TEXT NOT NULL UNIQUE,\n",
+ " ts_fetch TIMESTAMPTZ NOT NULL DEFAULT NOW(),\n",
+ " status URL_STATUS NOT NULL DEFAULT 'raw' -- ,\n",
+ " -- status_wendy WENDY_STATUS DEFAULT NULL,\n",
+ " -- ts_wendy TIMESTAMPTZ DEFAULT NULL\n",
+ " );\n",
+ " CREATE INDEX idx_urls_status ON urls(status);\n",
+ " CREATE INDEX idx_urls_ts_fetch ON urls(ts_fetch);\n",
+ "\n",
+ " CREATE TABLE URLS_DUPLICATE (\n",
+ " id_url_canonical INTEGER REFERENCES URLS(id),\n",
+ " id_url_duplicated INTEGER REFERENCES URLS(id),\n",
+ " PRIMARY KEY (id_url_canonical, id_url_duplicated)\n",
+ " );\n",
+ "\n",
+ " CREATE TABLE FEED (\n",
+ " id SMALLSERIAL PRIMARY KEY,\n",
+ " rss_feed TEXT NOT NULL UNIQUE\n",
+ " );\n",
+ " CREATE TABLE WEBSITE_OF_INTEREST (\n",
+ " id SMALLSERIAL PRIMARY KEY,\n",
+ " url_host TEXT NOT NULL UNIQUE\n",
+ " );\n",
+ " CREATE TABLE SEARCH (\n",
+ " id SMALLSERIAL PRIMARY KEY,\n",
+ " keyword_search TEXT NOT NULL UNIQUE\n",
+ " );\n",
+ " CREATE TABLE SOURCE (\n",
+ " id SMALLSERIAL PRIMARY KEY,\n",
+ " source TEXT NOT NULL UNIQUE\n",
+ " );\n",
+ "\n",
+ " CREATE TABLE URLS_SOURCE (\n",
+ " id_url INTEGER REFERENCES URLS(id),\n",
+ " id_source SMALLINT REFERENCES SOURCE(id) ON UPDATE CASCADE ON DELETE RESTRICT, -- Source encodes search information\n",
+ " PRIMARY KEY(id_url, id_source)\n",
+ " );\n",
+ " CREATE INDEX idx_source ON urls_source(id_source);\n",
+ "\n",
+ " CREATE TABLE WEBSITE_TO_FILTER (\n",
+ " id SMALLSERIAL PRIMARY KEY,\n",
+ " url_host TEXT NOT NULL UNIQUE\n",
+ " );\n",
+ "\n",
+ " CREATE TABLE STATUS_PATTERN_MATCHING (\n",
+ " pattern TEXT PRIMARY KEY,\n",
+ " priority SMALLINT NOT NULL,\n",
+ " status URL_STATUS NOT NULL\n",
+ " );\n",
+ " \n",
+ " \n",
+ " CREATE TABLE URL_CONTENT (\n",
+ " id_url INTEGER REFERENCES URLS(id),\n",
+ " date_published TIMESTAMPTZ NOT NULL DEFAULT NOW(),\n",
+ " title TEXT,\n",
+ " description TEXT,\n",
+ " content TEXT,\n",
+ " tags TEXT[],\n",
+ " authors TEXT[],\n",
+ " image_urls TEXT[],\n",
+ " );\n",
+ " CREATE INDEX idx_tags ON URL_CONTENT USING GIN(tags);\n",
+ " CREATE INDEX idx_authors ON URL_CONTENT USING GIN(authors);\n",
+ " \"\"\")\n",
+ "\n",
+ " # Feeds\n",
+ " cur.execute( \"INSERT INTO FEED (rss_feed) VALUES ('https://api.missingkids.org/missingkids/servlet/XmlServlet?act=rss&LanguageCountry=en_US&orgPrefix=NCMC');\" )\n",
+ " # Websites of interest\n",
+ " cur.execute( \"INSERT INTO WEBSITE_OF_INTEREST (url_host) VALUES ('www.unicef.org');\" )\n",
+ " # Search keywords\n",
+ " cur.execute( \"INSERT INTO SEARCH (keyword_search) VALUES ('child abuse');\" )\n",
+ " # Domains to filter\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('yewtu.be');\" )\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('twitter.com');\" )\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('libreddit.de');\" )\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('youtube.com');\" )\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('tiktok.com');\" )\n",
+ " cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('radio.foxnews.com');\" )\n",
+ " # Status update based on pattern matching (with priority to apply in order)\n",
+ " cur.execute( \"INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES ('.*missingkids.org/poster/.*', 50, 'valid');\" )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if INSERT_SAMPLE_DATA:\n",
+ " # Connect to an existing database\n",
+ " with psycopg.connect(connection_info) as conn:\n",
+ " # Open a cursor to perform database operations\n",
+ " with conn.cursor() as cur:\n",
+ " # Autocommit at end of transaction (Atomic insert of URLs and sources)\n",
+ " with conn.transaction() as tx:\n",
+ " # Valid\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.foxnews.com/us/husband-ruby-franke-utah-mommy-blogger-convicted-child-abuse-regrets-wifes-fall-fame', 'valid')\")\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.bbc.com/news/articles/ckg843y8y7no', 'valid')\")\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.wilx.com/2025/03/05/lenawee-county-man-arrested-possessing-child-abuse-material/', 'valid')\")\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.dw.com/en/trauma-how-child-abuse-victims-deal-with-parenthood/a-71833895', 'valid')\")\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://nypost.com/2025/03/06/us-news/colorado-day-care-worker-hit-with-51-charges-of-child-abuse-harassment-for-slapping-toddler/', 'valid')\")\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.fox35orlando.com/news/tavares-police-florida-boys-10-9-abused-sheer-brutality', 'valid')\")\n",
+ " # Invalid\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.google.com', 'invalid')\")\n",
+ "\n",
+ " cur.execute(\"INSERT INTO SOURCE (source) values ('news.google.com')\")\n",
+ " cur.execute(\"INSERT INTO SOURCE (source) values ('qwant.com')\")\n",
+ "\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (1, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (2, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (3, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (4, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (5, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (6, 1)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (7, 1)\")\n",
+ "\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (1, 2)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (2, 2)\")\n",
+ " cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (3, 2)\")\n",
+ "\n",
+ " for j in range(15):\n",
+ " import time\n",
+ " time.sleep(1)\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_{}.org', 'invalid')\".format(j))\n",
+ " \n",
+ " # Long URLs \n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf_23dj9sdgj9sdgj8sdf8ds8f.html', 'invalid')\".format(j))\n",
+ " cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf.html', 'invalid')\".format(j))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pprint import pprint\n",
+ "\n",
+ "# Connect to an existing database\n",
+ "with psycopg.connect(connection_info) as conn:\n",
+ " # Open a cursor to perform database operations\n",
+ " with conn.cursor() as cur:\n",
+ " # Get tables\n",
+ " cur.execute(\"SELECT table_name FROM information_schema.tables WHERE table_schema='public';\")\n",
+ " tables = [t[0] for t in cur.fetchall()]\n",
+ "\n",
+ " for t in tables:\n",
+ " print(\"\\t\", t)\n",
+ " pprint( cur.execute(\"SELECT * FROM {} LIMIT 50;\".format(t)).fetchall() )"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/A_Development.ipynb b/A_Development.ipynb
new file mode 100644
index 0000000..1dfb07a
--- /dev/null
+++ b/A_Development.ipynb
@@ -0,0 +1,247 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ollama\n",
+ "\n",
+ "#model = \"llama3.2:1b\"\n",
+ "client = ollama.Client(\n",
+ " host = 'https://ollamamodel.matitos.org',\n",
+ ")\n",
+ "l = client.list()\n",
+ "list_models = [m.get(\"model\") for m in l.model_dump().get(\"models\")]\n",
+ "\n",
+ "list_models"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for m in list_models:\n",
+ " context_key = [ k for k in client.show(m).model_dump().get(\"modelinfo\").keys() if \"context_length\" in k]\n",
+ " if (len(context_key) != 1):\n",
+ " print(\"Problem!!!\")\n",
+ " print(m, client.show(m).model_dump().get(\"modelinfo\").get(context_key[0]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "text = \"...\"\n",
+ "model = \"falcon3:1b\"\n",
+ "\n",
+ "msg_content = {\n",
+ " \"role\": \"user\", \n",
+ " \"content\": text,\n",
+ "}\n",
+ "response = client.chat(model=model, messages=[msg_content], stream=False)\n",
+ "print(response[\"message\"][\"content\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "import cv2\n",
+ "import base64\n",
+ "import numpy as np\n",
+ "\n",
+ "endpoint = \"http://192.168.2.64:12343/image\"\n",
+ "\n",
+ "\n",
+ "\n",
+ "prompt = \"Majestic mountain landscape with snow-capped peaks, autumn foliage in vibrant reds and oranges, a turquoise river winding through a valley, crisp and serene atmosphere, ultra-realistic style.\"\n",
+ "prompt = \"A group of kids happily playing in a joy environment\"\n",
+ "#prompt = \"A bitcoin behaving like a king, surrounded by small alternative coins. Detailed, geometric style\"\n",
+ "\n",
+ "json = {\n",
+ " \"prompt\": prompt,\n",
+ " \"num_inference_steps\": 10,\n",
+ " \"size\": \"512x512\",\n",
+ " \"seed\": 123456,\n",
+ "}\n",
+ "\n",
+ "for inf_step in [1, 4, 10, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100]:\n",
+ " json[\"num_inference_steps\"] = inf_step\n",
+ "\n",
+ " %time r = requests.post(endpoint, json=json)\n",
+ " print(\"Status code\", r.status_code)\n",
+ "\n",
+ " # Image\n",
+ " png_as_np = np.frombuffer(base64.b64decode(r.text), dtype=np.uint8)\n",
+ " image_bgr = cv2.imdecode(png_as_np, cv2.IMREAD_COLOR)\n",
+ "\n",
+ " cv2.imwrite(\"sample_img_{}.png\".format(json[\"num_inference_steps\"]), image_bgr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install trafilatura trafilatura[all] cchardet\n",
+ "import courlan\n",
+ "url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
+ "courlan.check_url(url)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install trafilatura\n",
+ "import trafilatura\n",
+ "from pprint import pprint\n",
+ "\n",
+ "url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
+ "url = \"https://www.missingkids.org/poster/USVA/VA25-0820/1\"\n",
+ "\n",
+ "# Fetch\n",
+ "doc = trafilatura.fetch_url(url)\n",
+ "# Content & metadata\n",
+ "metadata = trafilatura.extract_metadata(doc)\n",
+ "content = trafilatura.extract(doc)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pprint(metadata.as_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(content)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install newspaper4k"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import newspaper\n",
+ "\n",
+ "url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
+ "url = \"https://www.missingkids.org/poster/USVA/VA25-0820/1\"\n",
+ "\n",
+ "article = newspaper.article(url)\n",
+ "\n",
+ "url_photo = set([i for i in article.images if \"api.missingkids.org/photographs\" in i])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# !pip install news-please\n",
+ "from newsplease import NewsPlease\n",
+ "\n",
+ "url = \"https://variety.com/2025/film/news/gene-hackman-death-suspicious-gas-leak-search-warrant-1236322610/\"\n",
+ "url = \"https://www.bbc.com/news/articles/cewkkkvkzn9o\"\n",
+ "url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
+ "article = NewsPlease.from_url(url)\n",
+ "print(article.title)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(article.maintext)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "matitos",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 0000000..d7271c1
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+
+ matitos_db:
+ image: postgres:17
+ container_name: db_postgres
+ restart: unless-stopped
+ # Set shared memory limit when using docker-compose
+ shm_size: 128mb
+ environment:
+ POSTGRES_PASSWORD: ${DB_PASSWORD:-supermatitos}
+ POSTGRES_USER: ${DB_USERNAME:-supermatitos}
+ POSTGRES_DB: ${DB_DATABASE_NAME:-matitos}
+ POSTGRES_INITDB_ARGS: '--data-checksums'
+ #volumes:
+ # - ${PATH_BASE:-.}/postgres:/var/lib/postgresql/data
+ ports:
+ - 5432:5432
+
+# django:
+# Env: DB_HOST=matitos_db
+# DJANGO_DB_NAME=${DB_DATABASE_NAME:-matitos}
+# DJANGO_DB_USER=${DB_USERNAME:-supermatitos}
+# DJANGO_DB_PASSWORD=${DB_PASSWORD:-supermatitos}
+# DJANGO_DB_HOST=${DB_HOST:-localhost}
+# DJANGO_DB_PORT=${DB_PORT:-5432}
diff --git a/image_generation_app/Dockerfile b/image_generation_app/Dockerfile
new file mode 100644
index 0000000..a4feb2a
--- /dev/null
+++ b/image_generation_app/Dockerfile
@@ -0,0 +1,36 @@
+FROM continuumio/anaconda3
+
+# Based on:
+# https://www.reddit.com/r/StableDiffusion/comments/1gxbwp1/npu_accelerated_sd15_lcm_on_130_rk3588_sbc_30/
+
+WORKDIR /home
+
+RUN apt-get update && \
+ apt-get install git-lfs && \
+ # RKNN lib
+ git clone https://github.com/airockchip/rknn-toolkit2.git && \
+ cp rknn-toolkit2/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so /usr/lib && \
+ # Stable Diffusion
+ git clone https://huggingface.co/happyme531/Stable-Diffusion-1.5-LCM-ONNX-RKNN2 && \
+ # Dependencies
+ pip install diffusers pillow "numpy<2" rknn-toolkit-lite2 torch transformers
+
+WORKDIR /home/Stable-Diffusion-1.5-LCM-ONNX-RKNN2
+
+# FastAPI
+RUN conda install -c conda-forge libgl
+RUN pip install fastapi[standard] opencv-python
+
+COPY ./app /home/app
+
+# Replace writing image path
+RUN sed -i '/return os.path.join(out_folder, out_fname + ".png")/c \ \ \ \ return "images/image.png"' ./run_rknn-lcm.py
+RUN sed -i '/os.makedirs(out_folder, exist_ok=True)/c \ \ \ \ os.makedirs("images", exist_ok=True)' ./run_rknn-lcm.py
+# Multi core NPU
+RUN sed -i 's/RKNNLite.NPU_CORE_AUTO/RKNNLite.NPU_CORE_0_1_2/g' ./run_rknn-lcm.py
+
+# CMD ["/bin/bash"]
+CMD ["fastapi", "run", "/home/app/main.py", "--port", "80"]
+
+# docker build -t image_generation .
+# docker run --rm --privileged --device /dev/rknpu:/dev/rknpu --device /dev/dri:/dev/dri --security-opt systempaths=unconfined -p 12343:80 image_generation
diff --git a/image_generation_app/app/__init__.py b/image_generation_app/app/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/image_generation_app/app/main.py b/image_generation_app/app/main.py
new file mode 100644
index 0000000..72c45c1
--- /dev/null
+++ b/image_generation_app/app/main.py
@@ -0,0 +1,48 @@
+from fastapi import FastAPI, Response
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+import cv2
+import subprocess
+import base64
+
+class Item(BaseModel):
+ prompt: str | None = None
+ size: str | None = "512x512"
+ num_inference_steps: int | None = 4
+ seed: int | None = 123456
+
+def generate_image(item):
+ print(item)
+ # Parameters
+ seed = item.seed
+ num_inference_steps = item.num_inference_steps
+ size = item.size
+ prompt = item.prompt
+ command = 'python ./run_rknn-lcm.py --seed {} -i ./model -o ./images --num-inference-steps {} -s {} --prompt "{}"'.format(seed, num_inference_steps, size, prompt)
+
+ # Inference
+ output = subprocess.run(command, shell=True, capture_output=True)
+ print(output, "\n")
+
+ # Path to image
+ path_img = "./images/image.png" # glob.glob("./images/*")[0]
+ # Read
+ img = cv2.imread(path_img)
+ return img
+
+app = FastAPI()
+
+
+@app.get("/")
+def read_root():
+ return {"Hello": "World"}
+
+@app.post("/image")
+def get_image(item: Item):
+ # Generate
+ image = generate_image(item)
+ # Encode
+ retval, buffer = cv2.imencode('.png', image)
+ png_as_text = base64.b64encode(buffer)
+ # Return
+ return Response(png_as_text)
diff --git a/web_app/manage.py b/web_app/manage.py
new file mode 100755
index 0000000..a7da667
--- /dev/null
+++ b/web_app/manage.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+ """Run administrative tasks."""
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
+ try:
+ from django.core.management import execute_from_command_line
+ except ImportError as exc:
+ raise ImportError(
+ "Couldn't import Django. Are you sure it's installed and "
+ "available on your PYTHONPATH environment variable? Did you "
+ "forget to activate a virtual environment?"
+ ) from exc
+ execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/web_app/mysite/__init__.py b/web_app/mysite/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/web_app/mysite/__pycache__/__init__.cpython-312.pyc b/web_app/mysite/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000..a92596e
Binary files /dev/null and b/web_app/mysite/__pycache__/__init__.cpython-312.pyc differ
diff --git a/web_app/mysite/__pycache__/settings.cpython-312.pyc b/web_app/mysite/__pycache__/settings.cpython-312.pyc
new file mode 100644
index 0000000..664ce9c
Binary files /dev/null and b/web_app/mysite/__pycache__/settings.cpython-312.pyc differ
diff --git a/web_app/mysite/__pycache__/urls.cpython-312.pyc b/web_app/mysite/__pycache__/urls.cpython-312.pyc
new file mode 100644
index 0000000..674804b
Binary files /dev/null and b/web_app/mysite/__pycache__/urls.cpython-312.pyc differ
diff --git a/web_app/mysite/__pycache__/wsgi.cpython-312.pyc b/web_app/mysite/__pycache__/wsgi.cpython-312.pyc
new file mode 100644
index 0000000..4991a01
Binary files /dev/null and b/web_app/mysite/__pycache__/wsgi.cpython-312.pyc differ
diff --git a/web_app/mysite/asgi.py b/web_app/mysite/asgi.py
new file mode 100644
index 0000000..44c7dff
--- /dev/null
+++ b/web_app/mysite/asgi.py
@@ -0,0 +1,16 @@
+"""
+ASGI config for mysite project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
+
+application = get_asgi_application()
diff --git a/web_app/mysite/settings.py b/web_app/mysite/settings.py
new file mode 100644
index 0000000..08fc4eb
--- /dev/null
+++ b/web_app/mysite/settings.py
@@ -0,0 +1,132 @@
+"""
+Django settings for mysite project.
+
+Generated by 'django-admin startproject' using Django 5.1.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.1/ref/settings/
+"""
+
+import os
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-0+jg0u+%s@sj759i7@jn*%-#jl)8=siclb5908pwe!7=*$qb'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+ 'news.apps.NewsConfig',
+ 'django.contrib.admin',
+ 'django.contrib.auth',
+ 'django.contrib.contenttypes',
+ 'django.contrib.sessions',
+ 'django.contrib.messages',
+ 'django.contrib.staticfiles',
+]
+
+MIDDLEWARE = [
+ 'django.middleware.security.SecurityMiddleware',
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'mysite.urls'
+
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [],
+ 'APP_DIRS': True,
+ 'OPTIONS': {
+ 'context_processors': [
+ 'django.template.context_processors.debug',
+ 'django.template.context_processors.request',
+ 'django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages',
+ ],
+ },
+ },
+]
+
+WSGI_APPLICATION = 'mysite.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.postgresql',
+ 'NAME': os.environ.get("DJANGO_DB_NAME", "matitos"),
+ 'USER': os.environ.get("DJANGO_DB_USER", "supermatitos"),
+ 'PASSWORD': os.environ.get("DJANGO_DB_PASSWORD", "supermatitos"),
+ 'HOST': os.environ.get("DJANGO_DB_HOST", "localhost"),
+ 'PORT': os.environ.get("DJANGO_DB_PORT", "5432"),
+ #'OPTIONS': {
+ # 'options': '-c default_transaction_read_only=on'
+ #}
+ }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+ {
+ 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+ },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.1/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.1/howto/static-files/
+
+STATIC_URL = 'static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
diff --git a/web_app/mysite/urls.py b/web_app/mysite/urls.py
new file mode 100644
index 0000000..2ce5a4c
--- /dev/null
+++ b/web_app/mysite/urls.py
@@ -0,0 +1,26 @@
+"""
+URL configuration for mysite project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+ https://docs.djangoproject.com/en/5.1/topics/http/urls/
+Examples:
+Function views
+ 1. Add an import: from my_app import views
+ 2. Add a URL to urlpatterns: path('', views.home, name='home')
+Class-based views
+ 1. Add an import: from other_app.views import Home
+ 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
+Including another URLconf
+ 1. Import the include() function: from django.urls import include, path
+ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import include, path
+from django.views.generic.base import RedirectView
+
+urlpatterns = [
+ path("", RedirectView.as_view(url='news/', permanent=False)),
+ path("news/", include("news.urls")),
+ path('admin/', admin.site.urls),
+ # path("facerecognition", include("facerecognition.urls")),
+]
diff --git a/web_app/mysite/wsgi.py b/web_app/mysite/wsgi.py
new file mode 100644
index 0000000..61b0d9d
--- /dev/null
+++ b/web_app/mysite/wsgi.py
@@ -0,0 +1,16 @@
+"""
+WSGI config for mysite project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
+
+application = get_wsgi_application()
diff --git a/web_app/news/__init__.py b/web_app/news/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/web_app/news/__pycache__/__init__.cpython-312.pyc b/web_app/news/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000..66b2a3d
Binary files /dev/null and b/web_app/news/__pycache__/__init__.cpython-312.pyc differ
diff --git a/web_app/news/__pycache__/admin.cpython-312.pyc b/web_app/news/__pycache__/admin.cpython-312.pyc
new file mode 100644
index 0000000..a55392e
Binary files /dev/null and b/web_app/news/__pycache__/admin.cpython-312.pyc differ
diff --git a/web_app/news/__pycache__/apps.cpython-312.pyc b/web_app/news/__pycache__/apps.cpython-312.pyc
new file mode 100644
index 0000000..42dd93a
Binary files /dev/null and b/web_app/news/__pycache__/apps.cpython-312.pyc differ
diff --git a/web_app/news/__pycache__/models.cpython-312.pyc b/web_app/news/__pycache__/models.cpython-312.pyc
new file mode 100644
index 0000000..6a74e6a
Binary files /dev/null and b/web_app/news/__pycache__/models.cpython-312.pyc differ
diff --git a/web_app/news/__pycache__/urls.cpython-312.pyc b/web_app/news/__pycache__/urls.cpython-312.pyc
new file mode 100644
index 0000000..56a1f9b
Binary files /dev/null and b/web_app/news/__pycache__/urls.cpython-312.pyc differ
diff --git a/web_app/news/__pycache__/views.cpython-312.pyc b/web_app/news/__pycache__/views.cpython-312.pyc
new file mode 100644
index 0000000..a1cb7a3
Binary files /dev/null and b/web_app/news/__pycache__/views.cpython-312.pyc differ
diff --git a/web_app/news/admin.py b/web_app/news/admin.py
new file mode 100644
index 0000000..4ff3abf
--- /dev/null
+++ b/web_app/news/admin.py
@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+# Register your models here.
+
+from .models import Urls, UrlsSource, Source
+
+admin.site.register(Urls)
+admin.site.register(UrlsSource)
+admin.site.register(Source)
diff --git a/web_app/news/apps.py b/web_app/news/apps.py
new file mode 100644
index 0000000..44db8e5
--- /dev/null
+++ b/web_app/news/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class NewsConfig(AppConfig):
+ default_auto_field = 'django.db.models.BigAutoField'
+ name = 'news'
diff --git a/web_app/news/migrations/0001_initial.py b/web_app/news/migrations/0001_initial.py
new file mode 100644
index 0000000..320ab7a
--- /dev/null
+++ b/web_app/news/migrations/0001_initial.py
@@ -0,0 +1,38 @@
+# Generated by Django 5.1.6 on 2025-02-20 15:36
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='SOURCE',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('source', models.TextField()),
+ ],
+ ),
+ migrations.CreateModel(
+ name='URL',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('url', models.TextField()),
+ ('pub_date', models.DateTimeField(verbose_name='date published')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='URL_SOURCE',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('source', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='news.source')),
+ ('url', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='news.url')),
+ ],
+ ),
+ ]
diff --git a/web_app/news/migrations/0002_alter_source_table_alter_url_table_and_more.py b/web_app/news/migrations/0002_alter_source_table_alter_url_table_and_more.py
new file mode 100644
index 0000000..b88c65b
--- /dev/null
+++ b/web_app/news/migrations/0002_alter_source_table_alter_url_table_and_more.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.1.6 on 2025-02-20 16:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('news', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterModelTable(
+ name='source',
+ table='source',
+ ),
+ migrations.AlterModelTable(
+ name='url',
+ table='urls',
+ ),
+ migrations.AlterModelTable(
+ name='url_source',
+ table='urls_source',
+ ),
+ ]
diff --git a/web_app/news/migrations/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.py b/web_app/news/migrations/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.py
new file mode 100644
index 0000000..b65316d
--- /dev/null
+++ b/web_app/news/migrations/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 5.1.6 on 2025-02-20 16:18
+
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('news', '0002_alter_source_table_alter_url_table_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='url',
+ name='pub_date',
+ ),
+ migrations.AddField(
+ model_name='url',
+ name='status',
+ field=models.CharField(choices=[('raw', 'Raw'), ('error', 'Error'), ('valid', 'Valid'), ('unknown', 'Unknown'), ('invalid', 'Invalid'), ('duplicate', 'Duplicate')], default='raw'),
+ ),
+ migrations.AddField(
+ model_name='url',
+ name='ts_fetch',
+ field=models.DateTimeField(db_default=django.db.models.functions.datetime.Now(), verbose_name='Date fetched'),
+ ),
+ migrations.AlterField(
+ model_name='url',
+ name='url',
+ field=models.TextField(verbose_name='URL'),
+ ),
+ ]
diff --git a/web_app/news/migrations/0004_alter_url_source_unique_together.py b/web_app/news/migrations/0004_alter_url_source_unique_together.py
new file mode 100644
index 0000000..c9900b5
--- /dev/null
+++ b/web_app/news/migrations/0004_alter_url_source_unique_together.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.1.6 on 2025-02-20 16:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('news', '0003_remove_url_pub_date_url_status_url_ts_fetch_and_more'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='url_source',
+ unique_together={('url', 'source')},
+ ),
+ ]
diff --git a/web_app/news/migrations/0005_urls_remove_url_source_url_and_more.py b/web_app/news/migrations/0005_urls_remove_url_source_url_and_more.py
new file mode 100644
index 0000000..b720796
--- /dev/null
+++ b/web_app/news/migrations/0005_urls_remove_url_source_url_and_more.py
@@ -0,0 +1,59 @@
+# Generated by Django 5.1.6 on 2025-02-20 16:53
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('news', '0004_alter_url_source_unique_together'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Urls',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('url', models.TextField(unique=True)),
+ ('ts_fetch', models.DateTimeField()),
+ ('status', models.TextField(choices=[('raw', 'Raw'), ('error', 'Error'), ('valid', 'Valid'), ('unknown', 'Unknown'), ('invalid', 'Invalid'), ('duplicate', 'Duplicate')], default='raw')),
+ ],
+ options={
+ 'db_table': 'urls',
+ 'managed': False,
+ },
+ ),
+ migrations.RemoveField(
+ model_name='url_source',
+ name='url',
+ ),
+ migrations.AlterUniqueTogether(
+ name='url_source',
+ unique_together=None,
+ ),
+ migrations.RemoveField(
+ model_name='url_source',
+ name='source',
+ ),
+ migrations.AlterModelOptions(
+ name='source',
+ options={'managed': False},
+ ),
+ migrations.CreateModel(
+ name='UrlsSource',
+ fields=[
+ ('id_url', models.OneToOneField(db_column='id_url', on_delete=django.db.models.deletion.DO_NOTHING, primary_key=True, serialize=False, to='news.urls')),
+ ],
+ options={
+ 'db_table': 'urls_source',
+ 'managed': False,
+ },
+ ),
+ migrations.DeleteModel(
+ name='URL',
+ ),
+ migrations.DeleteModel(
+ name='URL_SOURCE',
+ ),
+ ]
diff --git a/web_app/news/migrations/0006_alter_urls_options.py b/web_app/news/migrations/0006_alter_urls_options.py
new file mode 100644
index 0000000..be21c9d
--- /dev/null
+++ b/web_app/news/migrations/0006_alter_urls_options.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.1.6 on 2025-03-06 09:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('news', '0005_urls_remove_url_source_url_and_more'),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name='urls',
+ options={'managed': False, 'ordering': ['-ts_fetch']},
+ ),
+ ]
diff --git a/web_app/news/migrations/__init__.py b/web_app/news/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/web_app/news/migrations/__pycache__/0001_initial.cpython-312.pyc b/web_app/news/migrations/__pycache__/0001_initial.cpython-312.pyc
new file mode 100644
index 0000000..922532d
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0001_initial.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/0002_alter_source_table_alter_url_table_and_more.cpython-312.pyc b/web_app/news/migrations/__pycache__/0002_alter_source_table_alter_url_table_and_more.cpython-312.pyc
new file mode 100644
index 0000000..9739c13
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0002_alter_source_table_alter_url_table_and_more.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.cpython-312.pyc b/web_app/news/migrations/__pycache__/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.cpython-312.pyc
new file mode 100644
index 0000000..4832345
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0003_remove_url_pub_date_url_status_url_ts_fetch_and_more.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/0004_alter_url_source_unique_together.cpython-312.pyc b/web_app/news/migrations/__pycache__/0004_alter_url_source_unique_together.cpython-312.pyc
new file mode 100644
index 0000000..c942a86
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0004_alter_url_source_unique_together.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/0005_urls_remove_url_source_url_and_more.cpython-312.pyc b/web_app/news/migrations/__pycache__/0005_urls_remove_url_source_url_and_more.cpython-312.pyc
new file mode 100644
index 0000000..14e41fc
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0005_urls_remove_url_source_url_and_more.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/0006_alter_urls_options.cpython-312.pyc b/web_app/news/migrations/__pycache__/0006_alter_urls_options.cpython-312.pyc
new file mode 100644
index 0000000..5e02235
Binary files /dev/null and b/web_app/news/migrations/__pycache__/0006_alter_urls_options.cpython-312.pyc differ
diff --git a/web_app/news/migrations/__pycache__/__init__.cpython-312.pyc b/web_app/news/migrations/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000..13a8e98
Binary files /dev/null and b/web_app/news/migrations/__pycache__/__init__.cpython-312.pyc differ
diff --git a/web_app/news/models.py b/web_app/news/models.py
new file mode 100644
index 0000000..7fa4692
--- /dev/null
+++ b/web_app/news/models.py
@@ -0,0 +1,46 @@
+from django.db import models
+
+# Create your models here.
+class Urls(models.Model):
+ class STATUS_ENUM(models.TextChoices):
+ RAW = "raw"
+ ERROR = "error"
+ VALID = "valid"
+ UNKNOWN = "unknown"
+ INVALID = "invalid"
+ DUPLICATE = "duplicate"
+
+ url = models.TextField(unique=True)
+ ts_fetch = models.DateTimeField()
+ status = models.TextField(choices=STATUS_ENUM, default=STATUS_ENUM.RAW) # This field type is a guess.
+
+ def __str__(self):
+ return self.url
+
+ class Meta:
+ managed = False
+ db_table = 'urls' # db_table = '{}_urls'.format(project_name)
+ ordering = ["-ts_fetch"]
+
+class Source(models.Model):
+ id = models.SmallAutoField(primary_key=True)
+ source = models.TextField(unique=True)
+
+ def __str__(self):
+ return self.source
+
+ class Meta:
+ managed = False
+ db_table = 'source'
+
+class UrlsSource(models.Model):
+ id_url = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url', primary_key=True) # The composite primary key (id_url, id_source) found, that is not supported. The first column is selected.
+ id_source = models.ForeignKey(Source, models.DO_NOTHING, db_column='id_source')
+
+ def __str__(self):
+ return "Source: {}, URL: {}".format(self.id_source, self.id_url)
+
+ class Meta:
+ managed = False
+ db_table = 'urls_source'
+ unique_together = (('id_url', 'id_source'),)
diff --git a/web_app/news/templates/item_list.html b/web_app/news/templates/item_list.html
new file mode 100644
index 0000000..f33e579
--- /dev/null
+++ b/web_app/news/templates/item_list.html
@@ -0,0 +1,508 @@
+
+
+
+
+
+ News
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {% include 'item_list_partial.html' %}
+
+
+
+
+
+
+
+
diff --git a/web_app/news/templates/item_list_partial.html b/web_app/news/templates/item_list_partial.html
new file mode 100644
index 0000000..da2fe67
--- /dev/null
+++ b/web_app/news/templates/item_list_partial.html
@@ -0,0 +1,87 @@
+{% load custom_filters %}
+
+
+
+
+
+ | URL |
+ Fetch date |
+ Sources |
+ Status |
+ Action |
+
+
+
+ {% for item in page_obj %}
+
+ | {{ item.url }} |
+ {{ item.ts_fetch }} |
+
+ {% with sources_map|dict_get:item.id as sources %}
+ {% if sources %}
+ {% for source in sources %}
+ {{ source }}
+ {% endfor %}
+ {% else %}
+ No sources
+ {% endif %}
+ {% endwith %}
+ |
+
+ {% if item.status == 'raw' %}
+ {{ item.status|capfirst }}
+ {% elif item.status == 'error' %}
+ {{ item.status|capfirst }}
+ {% elif item.status == 'valid' %}
+ {{ item.status|capfirst }}
+ {% elif item.status == 'unknown' %}
+ {{ item.status|capfirst }}
+ {% elif item.status == 'invalid' %}
+ {{ item.status|capfirst }}
+ {% elif item.status == 'duplicate' %}
+ {{ item.status|capfirst }}
+ {% else %}
+ Unknown
+ {% endif %}
+ |
+
+ Details
+ |
+
+
+ {% empty %}
+
+ | No items available. |
+
+ {% endfor %}
+
+
+
+
+
+
+
diff --git a/web_app/news/templates/url_detail.html b/web_app/news/templates/url_detail.html
new file mode 100644
index 0000000..8b8a7bc
--- /dev/null
+++ b/web_app/news/templates/url_detail.html
@@ -0,0 +1,188 @@
+
+
+
+
+
+ {% block title %}News{% endblock %}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
URL Details
+
+
+ | URL |
+ {{ url_item.url }} |
+
+
+ | Fetch Date |
+ {{ url_item.ts_fetch }} |
+
+
+ | Sources |
+ {{ sources|join:", " }} |
+
+
+ | Status |
+ {{ url_item.status }} |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Loading...
+
+
+
+
+
+
+
+ {% block extra_js %}{% endblock %}
+
+
diff --git a/web_app/news/templatetags/__init__.py b/web_app/news/templatetags/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/web_app/news/templatetags/__pycache__/__init__.cpython-312.pyc b/web_app/news/templatetags/__pycache__/__init__.cpython-312.pyc
new file mode 100644
index 0000000..6ac09fa
Binary files /dev/null and b/web_app/news/templatetags/__pycache__/__init__.cpython-312.pyc differ
diff --git a/web_app/news/templatetags/__pycache__/custom_filters.cpython-312.pyc b/web_app/news/templatetags/__pycache__/custom_filters.cpython-312.pyc
new file mode 100644
index 0000000..45f8fe2
Binary files /dev/null and b/web_app/news/templatetags/__pycache__/custom_filters.cpython-312.pyc differ
diff --git a/web_app/news/templatetags/custom_filters.py b/web_app/news/templatetags/custom_filters.py
new file mode 100644
index 0000000..f4ad62b
--- /dev/null
+++ b/web_app/news/templatetags/custom_filters.py
@@ -0,0 +1,8 @@
+from django import template
+
+register = template.Library()
+
+@register.filter
+def dict_get(dictionary, key):
+ """Custom filter to get a value from a dictionary in Django templates."""
+ return dictionary.get(key, [])
diff --git a/web_app/news/tests.py b/web_app/news/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/web_app/news/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/web_app/news/urls.py b/web_app/news/urls.py
new file mode 100644
index 0000000..a92a8ee
--- /dev/null
+++ b/web_app/news/urls.py
@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+ path("", views.news, name="home"),
+ path('url//', views.url_detail_view, name='url_detail'),
+ path('url//fetch/', views.fetch_details, name='fetch_details'),]
diff --git a/web_app/news/views.py b/web_app/news/views.py
new file mode 100644
index 0000000..fb02b0f
--- /dev/null
+++ b/web_app/news/views.py
@@ -0,0 +1,97 @@
+from django.http import StreamingHttpResponse, HttpResponse, JsonResponse
+from django.shortcuts import render, get_object_or_404
+from django.core.paginator import Paginator
+import requests
+from django.http import StreamingHttpResponse
+import json
+import time
+import ollama
+
+from .models import Urls, Source, UrlsSource
+
+# Create your views here.
+def index(request):
+ return HttpResponse("Hello, world. You're at the news index.")
+
+def news(request):
+ # URLs
+ urls = Urls.objects.all()
+ # Sources
+ sources = Source.objects.all()
+
+ # Parameters
+ page_number = request.GET.get("page", 1)
+ num_items = request.GET.get("items", 15)
+ source_ids = request.GET.get("sources", ','.join([str(s.id) for s in sources]))
+ status_filters = request.GET.get("status", None)
+
+ # Filters
+ if (status_filters) and (status_filters != "all"):
+ urls = urls.filter(status__in=status_filters.split(","))
+ if (source_ids) and (source_ids != "all"):
+ # TODO: Distinct needed?
+ urls = urls.filter(urlssource__id_source__in=source_ids.split(",")).distinct()
+
+ # Pagination
+ paginator = Paginator(urls, num_items)
+ page_obj = paginator.get_page(page_number)
+
+ # Map URL IDs to their sources, only for subset of URLs (page of interest)
+ sources_map = {
+ url.id: list(Source.objects.filter(urlssource__id_url=url).values_list('source', flat=True))
+ for url in page_obj.object_list
+ }
+
+ context = {
+ "page_obj": page_obj,
+ "sources": sources,
+ "sources_map": sources_map,
+ "list_status": Urls.STATUS_ENUM.values,
+ "list_urls_per_page": [15, 50, 100],
+ }
+
+ # If request is AJAX, return JSON response
+ if request.headers.get("X-Requested-With") == "XMLHttpRequest":
+ return JsonResponse({'items_html': render(request, 'item_list_partial.html', context).content.decode('utf-8')})
+
+ return render(request, "item_list.html", context)
+
+
+def url_detail_view(request, id):
+ url_item = get_object_or_404(Urls, id=id)
+ url_sources = list(Source.objects.filter(urlssource__id_url=url_item).values_list('source', flat=True))
+
+ # TODO: https://github.com/ollama/ollama-python?tab=readme-ov-file#async-client
+ # LLM models available
+ client = ollama.Client(host = 'https://ollamamodel.matitos.org')
+ models = [m.model for m in client.list().models]
+
+ context = {
+ 'url_item': url_item,
+ 'sources': url_sources,
+ 'models': models,
+ "prompt": "Provide in one paragraph the what, why, when, where, who, and how of the content below. Also provide a one paragraph summary of the content:",
+ #"prompt": "Image you are a journalist, TLDR in a paragraph:",
+ #"prompt": "Below you will find the whole content of a news article:\n{}\nProvide a concise summary of one paragraph maximum of the content.".format(content)
+ }
+ return render(request, 'url_detail.html', context)
+
+def fetch_details(request, id):
+ url_item = get_object_or_404(Urls, id=id)
+ url_param = request.GET.get("url", "") # Get URL
+ model = request.GET.get("model", "") # Get LLM model
+ text = request.GET.get("text", "") # Get LLM prompt
+
+ # LLM
+ client = ollama.Client(host = 'https://ollamamodel.matitos.org')
+
+ def stream_response():
+ msg_content = {
+ "role": "user",
+ "content": text,
+ }
+ response = client.chat(model=model, messages=[msg_content], stream=True)
+ for chunk in response:
+ yield chunk["message"]["content"] # Stream each chunk of text
+
+ return StreamingHttpResponse(stream_response(), content_type="text/plain")