General search fix, status pattern match regex, find feeds on startup

app_urls/fetcher/views_base.py

@@ -0,0 +1,74 @@
+import os
+import psycopg
+from .tasks import background_task
+from django.http import JsonResponse, HttpResponse
+
+####################################################################################################
+def trigger_task(request, task):
+    # Enqueue function in "default" queue
+    background_task.delay(task)  
+    return JsonResponse({"message": "Task has been enqueued!", "task": task})
+
+####################################################################################################
+def link_list(request):
+    # Base URL path
+    app_url = request.build_absolute_uri()
+    # Tasks
+    links_fetch = ["fetch_feeds", "fetch_parser", "fetch_search", "fetch_missingkids_5", "fetch_missingkids_all"]
+    links_process = ["process_raw_urls_50", "process_error_urls_50", "process_missing_kids_urls_50", "process_missing_kids_urls_all"]
+    # List of links
+    list_links = \
+        [ os.path.join(app_url, "admin"), os.path.join(app_url, "urls") ] + \
+        [ os.path.join(app_url, "logs", log_type) for log_type in ["database", "debug", "info", "warning"] ] + \
+        [ os.path.join(app_url, "task", l) for l in links_fetch + links_process ]
+    # Json
+    return JsonResponse({"links": list_links })
+
+####################################################################################################
+def logs(request, log_type):
+    # Capture output: python manage.py rqstats
+    try:
+        with open( os.path.join( os.getenv("PATH_LOGS_DIRECTORY", "logs"), "{}.log".format(log_type) ), "r") as f:
+            file_content = f.read()
+    except Exception as e:
+        file_content = "Error reading logs for log type :{}".format(log_type)
+    return HttpResponse(file_content, content_type="text/plain")
+
+####################################################################################################
+def log_db(request):
+    # TODO: Django connection
+    connection_info = "host={} port={} dbname={} user={} password={} connect_timeout=60".format(
+        os.environ.get("DB_HOST", "localhost"),
+        os.environ.get("DB_PORT", "5432"),
+        os.environ.get("DB_NAME", "matitos"),
+        os.environ.get("DB_USER", "supermatitos"),
+        os.environ.get("DB_PASSWORD", "supermatitos")
+    )
+
+    # Connect to an existing database
+    with psycopg.connect(connection_info) as conn:
+        # Open a cursor to perform database operations
+        with conn.cursor() as cur:
+            # Create URLs table
+            r = cur.execute(""" 
+                SELECT
+                    relname AS "relation",
+                    pg_size_pretty (
+                        pg_total_relation_size (C .oid)
+                    ) AS "total_size"
+                FROM
+                    pg_class C
+                LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace)
+                WHERE
+                    nspname NOT IN (
+                        'pg_catalog',
+                        'information_schema'
+                    )
+                AND C .relkind <> 'i'
+                AND nspname !~ '^pg_toast'
+                ORDER BY
+                    pg_total_relation_size (C .oid) DESC
+                LIMIT 100;
+            """).fetchall()
+    return HttpResponse( "\n".join([str(e) for e in r]) )
+####################################################################################################