{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install psycopg[binary]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "db_postgres\n",
      "\u001b[1A\u001b[1B\u001b[0G\u001b[?25l[+] Running 1/0\n",
      " ⠿ Container db_postgres  \u001b[39mStarting\u001b[0m                                         \u001b[34m0.1s \u001b[0m\n",
      " \u001b[32m✔\u001b[0m Container db_redis     \u001b[32mRunning\u001b[0m                                          \u001b[34m0.0s \u001b[0m\n",
      "\u001b[?25h\u001b[1A\u001b[1A\u001b[1A\u001b[0G\u001b[?25l\u001b[34m[+] Running 2/2\u001b[0m\n",
      " \u001b[32m✔\u001b[0m Container db_postgres  \u001b[32mStarted\u001b[0m                                          \u001b[34m0.2s \u001b[0m\n",
      " \u001b[32m✔\u001b[0m Container db_redis     \u001b[32mRunning\u001b[0m                                          \u001b[34m0.0s \u001b[0m\n",
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!docker rm -f db_postgres; docker compose -f docker/docker-compose.yml up -d ; sleep 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "INSERT_TABLES = True\n",
    "INSERT_SAMPLE_DATA = True\n",
    "\n",
    "import psycopg\n",
    "connection_info = \"host={} port={} user={} password={} dbname={}\".format(\"localhost\", \"5432\", \"supermatitos\", \"supermatitos\", \"matitos\")\n",
    "\n",
    "from datetime import datetime, timezone\n",
    "\n",
    "\n",
    "if INSERT_TABLES:\n",
    "    # Connect to an existing database\n",
    "    with psycopg.connect(connection_info) as conn:\n",
    "        # Open a cursor to perform database operations\n",
    "        with conn.cursor() as cur:\n",
    "            # Autocommit at end of transaction (Atomic insert of URLs and sources)\n",
    "            with conn.transaction() as tx:\n",
    "                # Create URLs table\n",
    "                c = cur.execute(\"\"\"\n",
    "                    CREATE TYPE URL_STATUS AS ENUM ('raw', 'error', 'valid', 'unknown', 'invalid', 'duplicate');\n",
    "\n",
    "                    CREATE TABLE URLS (\n",
    "                        id SERIAL PRIMARY KEY,\n",
    "                        url TEXT NOT NULL UNIQUE,\n",
    "                        ts_fetch TIMESTAMPTZ NOT NULL DEFAULT NOW(),\n",
    "                        status URL_STATUS NOT NULL DEFAULT 'raw' -- ,\n",
    "                        -- status_wendy WENDY_STATUS DEFAULT NULL,\n",
    "                        -- ts_wendy TIMESTAMPTZ DEFAULT NULL\n",
    "                    );\n",
    "                    CREATE INDEX idx_urls_status ON urls(status);\n",
    "                    CREATE INDEX idx_urls_ts_fetch ON urls(ts_fetch);\n",
    "\n",
    "                    CREATE TABLE URLS_DUPLICATE (\n",
    "                        id_url_canonical INTEGER REFERENCES URLS(id),\n",
    "                        id_url_duplicated INTEGER REFERENCES URLS(id),\n",
    "                        PRIMARY KEY (id_url_canonical, id_url_duplicated)\n",
    "                    );\n",
    "\n",
    "                    CREATE TABLE FEED (\n",
    "                        id SMALLSERIAL PRIMARY KEY,\n",
    "                        rss_feed TEXT NOT NULL UNIQUE\n",
    "                    );\n",
    "                    CREATE TABLE WEBSITE_OF_INTEREST (\n",
    "                        id SMALLSERIAL PRIMARY KEY,\n",
    "                        url_host TEXT NOT NULL UNIQUE\n",
    "                    );\n",
    "                    CREATE TABLE SEARCH (\n",
    "                        id SMALLSERIAL PRIMARY KEY,\n",
    "                        keyword_search TEXT NOT NULL UNIQUE\n",
    "                    );\n",
    "                    CREATE TABLE SOURCE (\n",
    "                        id SMALLSERIAL PRIMARY KEY,\n",
    "                        source TEXT NOT NULL UNIQUE\n",
    "                    );\n",
    "\n",
    "                    CREATE TABLE URLS_SOURCE (\n",
    "                        id_url INTEGER REFERENCES URLS(id),\n",
    "                        id_source SMALLINT REFERENCES SOURCE(id) ON UPDATE CASCADE ON DELETE RESTRICT, -- Source encodes search information\n",
    "                        PRIMARY KEY(id_url, id_source)\n",
    "                    );\n",
    "                    CREATE INDEX idx_source ON urls_source(id_source);\n",
    "\n",
    "                    CREATE TABLE WEBSITE_TO_FILTER (\n",
    "                        id SMALLSERIAL PRIMARY KEY,\n",
    "                        url_host TEXT NOT NULL UNIQUE\n",
    "                    );\n",
    "\n",
    "                    CREATE TABLE STATUS_PATTERN_MATCHING (\n",
    "                        pattern TEXT PRIMARY KEY,\n",
    "                        priority SMALLINT NOT NULL,\n",
    "                        status URL_STATUS NOT NULL\n",
    "                    );\n",
    "                    \n",
    "                    \n",
    "                    CREATE TABLE URL_CONTENT (\n",
    "                        id_url INTEGER PRIMARY KEY REFERENCES URLS(id),\n",
    "                        date_published TIMESTAMPTZ DEFAULT NOW(),\n",
    "                        title TEXT,\n",
    "                        description TEXT,\n",
    "                        content TEXT,\n",
    "                        valid_content BOOLEAN,\n",
    "                        language CHAR(2), -- ISO 639-1 Code\n",
    "                        keywords TEXT[],\n",
    "                        tags TEXT[],\n",
    "                        authors TEXT[],\n",
    "                        image_main TEXT,\n",
    "                        images_url TEXT[],\n",
    "                        videos_url TEXT[],\n",
    "                        url_host TEXT,    -- www.breitbart.com\n",
    "                        site_name TEXT    -- Breitbart News\n",
    "                    );\n",
    "                    CREATE INDEX idx_tags ON URL_CONTENT USING GIN(tags);\n",
    "                    CREATE INDEX idx_authors ON URL_CONTENT USING GIN(authors);\n",
    "                    CREATE INDEX idx_date_published ON URL_CONTENT (date_published);\n",
    "                    CREATE INDEX idx_valid_content ON URL_CONTENT (valid_content);\n",
    "                    CREATE INDEX idx_language ON URL_CONTENT (language);\n",
    "                    CREATE INDEX idx_url_host ON URL_CONTENT (url_host);\n",
    "                \"\"\")\n",
    "\n",
    "                # Feeds\n",
    "                cur.execute( \"INSERT INTO FEED (rss_feed) VALUES ('https://api.missingkids.org/missingkids/servlet/XmlServlet?act=rss&LanguageCountry=en_US&orgPrefix=NCMC');\" )\n",
    "                # Websites of interest\n",
    "                cur.execute( \"INSERT INTO WEBSITE_OF_INTEREST (url_host) VALUES ('www.unicef.org');\" )\n",
    "                # Search keywords\n",
    "                cur.execute( \"INSERT INTO SEARCH (keyword_search) VALUES ('child abuse');\" )\n",
    "                # Domains to filter\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('yewtu.be');\" )\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('twitter.com');\" )\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('libreddit.de');\" )\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('youtube.com');\" )\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('tiktok.com');\" )\n",
    "                cur.execute( \"INSERT INTO WEBSITE_TO_FILTER (url_host) VALUES ('radio.foxnews.com');\" )\n",
    "                # Status update based on pattern matching (with priority to apply in order)\n",
    "                cur.execute( \"INSERT INTO STATUS_PATTERN_MATCHING (pattern, priority, status) VALUES ('.*missingkids.org/poster/.*', 50, 'valid');\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if INSERT_SAMPLE_DATA:\n",
    "    # Connect to an existing database\n",
    "    with psycopg.connect(connection_info) as conn:\n",
    "        # Open a cursor to perform database operations\n",
    "        with conn.cursor() as cur:\n",
    "            # Autocommit at end of transaction (Atomic insert of URLs and sources)\n",
    "                with conn.transaction() as tx:\n",
    "                    # Valid\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.foxnews.com/us/husband-ruby-franke-utah-mommy-blogger-convicted-child-abuse-regrets-wifes-fall-fame', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.bbc.com/news/articles/ckg843y8y7no', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.wilx.com/2025/03/05/lenawee-county-man-arrested-possessing-child-abuse-material/', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.dw.com/en/trauma-how-child-abuse-victims-deal-with-parenthood/a-71833895', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://nypost.com/2025/03/06/us-news/colorado-day-care-worker-hit-with-51-charges-of-child-abuse-harassment-for-slapping-toddler/', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.fox35orlando.com/news/tavares-police-florida-boys-10-9-abused-sheer-brutality', 'valid')\")\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('https://www.google.com', 'invalid')\")\n",
    "\n",
    "                    cur.execute(\"INSERT INTO SOURCE (source) values ('news.google.com')\")\n",
    "                    cur.execute(\"INSERT INTO SOURCE (source) values ('qwant.com')\")\n",
    "\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (1, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (2, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (3, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (4, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (5, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (6, 1)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (7, 1)\")\n",
    "\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (1, 2)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (2, 2)\")\n",
    "                    cur.execute(\"INSERT INTO URLS_SOURCE (id_url, id_source) values (3, 2)\")\n",
    "\n",
    "                    for j in range(15):\n",
    "                        import time\n",
    "                        time.sleep(1)\n",
    "                        cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_{}.org', 'invalid')\".format(j))\n",
    "                        \n",
    "                    # Long URLs                    \n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf_23dj9sdgj9sdgj8sdf8ds8f.html', 'invalid')\".format(j))\n",
    "                    cur.execute(\"INSERT INTO URLS (url, status) values ('www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf.html', 'invalid')\".format(j))\n",
    "\n",
    "                    # URL Content\n",
    "                    language, content = \"en\", \"Bla Bla Bla!!!\"*25\n",
    "                    cur.execute(\"INSERT INTO URL_CONTENT (id_url, date_published, title, description, content, language, tags, authors, image_urls) values (%s, %s, 'Mommy blogger turned child abuser', %s, 'Hello there!', %s, %s, %s, %s)\", \n",
    "                                (1, datetime.now(tz=timezone.utc), content, language, [\"child abuse\", \"social media\"], [\"Audrey Conklin\"], [\"https://a57.foxnews.com/static.foxnews.com/foxnews.com/content/uploads/2023/08/1440/810/image-58.jpg?ve=1&tl=1\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t urls\n",
      "[(1,\n",
      "  'https://www.foxnews.com/us/husband-ruby-franke-utah-mommy-blogger-convicted-child-abuse-regrets-wifes-fall-fame',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (2,\n",
      "  'https://www.bbc.com/news/articles/ckg843y8y7no',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (3,\n",
      "  'https://www.wilx.com/2025/03/05/lenawee-county-man-arrested-possessing-child-abuse-material/',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (4,\n",
      "  'https://www.dw.com/en/trauma-how-child-abuse-victims-deal-with-parenthood/a-71833895',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (5,\n",
      "  'https://nypost.com/2025/03/06/us-news/colorado-day-care-worker-hit-with-51-charges-of-child-abuse-harassment-for-slapping-toddler/',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (6,\n",
      "  'https://www.fox35orlando.com/news/tavares-police-florida-boys-10-9-abused-sheer-brutality',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'valid'),\n",
      " (7,\n",
      "  'https://www.google.com',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (8,\n",
      "  'www.super_0.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (9,\n",
      "  'www.super_1.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (10,\n",
      "  'www.super_2.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (11,\n",
      "  'www.super_3.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (12,\n",
      "  'www.super_4.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (13,\n",
      "  'www.super_5.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (14,\n",
      "  'www.super_6.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (15,\n",
      "  'www.super_7.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (16,\n",
      "  'www.super_8.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (17,\n",
      "  'www.super_9.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (18,\n",
      "  'www.super_10.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (19,\n",
      "  'www.super_11.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (20,\n",
      "  'www.super_12.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (21,\n",
      "  'www.super_13.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (22,\n",
      "  'www.super_14.org',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (23,\n",
      "  'www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf_23dj9sdgj9sdgj8sdf8ds8f.html',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid'),\n",
      " (24,\n",
      "  'www.super_url.org/superextrakmsdimsdf/349mvlsdfsdfwr/akivsdmimnsdifmisdf.html',\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 23, 32211, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'invalid')]\n",
      "\t urls_duplicate\n",
      "[]\n",
      "\t feed\n",
      "[(1,\n",
      "  'https://api.missingkids.org/missingkids/servlet/XmlServlet?act=rss&LanguageCountry=en_US&orgPrefix=NCMC')]\n",
      "\t website_of_interest\n",
      "[(1, 'www.unicef.org')]\n",
      "\t search\n",
      "[(1, 'child abuse')]\n",
      "\t urls_source\n",
      "[(1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1), (1, 2), (2, 2), (3, 2)]\n",
      "\t source\n",
      "[(1, 'news.google.com'), (2, 'qwant.com')]\n",
      "\t website_to_filter\n",
      "[(1, 'yewtu.be'),\n",
      " (2, 'twitter.com'),\n",
      " (3, 'libreddit.de'),\n",
      " (4, 'youtube.com'),\n",
      " (5, 'tiktok.com'),\n",
      " (6, 'radio.foxnews.com')]\n",
      "\t status_pattern_matching\n",
      "[('.*missingkids.org/poster/.*', 50, 'valid')]\n",
      "\t url_content\n",
      "[(1,\n",
      "  datetime.datetime(2025, 3, 7, 16, 57, 38, 54447, tzinfo=zoneinfo.ZoneInfo(key='Etc/UTC')),\n",
      "  'Mommy blogger turned child abuser',\n",
      "  'Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla '\n",
      "  'Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla '\n",
      "  'Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla '\n",
      "  'Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla '\n",
      "  'Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!Bla Bla Bla!!!',\n",
      "  'Hello there!',\n",
      "  ['child abuse', 'social media'],\n",
      "  ['Audrey Conklin'],\n",
      "  ['https://a57.foxnews.com/static.foxnews.com/foxnews.com/content/uploads/2023/08/1440/810/image-58.jpg?ve=1&tl=1'])]\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "# Connect to an existing database\n",
    "with psycopg.connect(connection_info) as conn:\n",
    "    # Open a cursor to perform database operations\n",
    "    with conn.cursor() as cur:\n",
    "        # Get tables\n",
    "        cur.execute(\"SELECT table_name FROM information_schema.tables WHERE table_schema='public';\")\n",
    "        tables = [t[0] for t in cur.fetchall()]\n",
    "\n",
    "        for t in tables:\n",
    "            print(\"\\t\", t)\n",
    "            pprint( cur.execute(\"SELECT * FROM {} LIMIT 50;\".format(t)).fetchall() )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "matitos",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}