Dockerization, whitenoise serving static, refactor

app_urls/fetcher/models.py

@@ -0,0 +1,140 @@
+from django.db import models
+from django.contrib.postgres.fields import ArrayField
+
+# Create your models here.
+class Search(models.Model):
+    class TYPE_ENUM(models.TextChoices):
+        RSS_FEED = "rss_feed", "RSS_Feed"
+        KEYWORD_SEARCH = "keyword_search", "Keyword_Search"
+        URL_HOST = "url_host", "URL_Host"
+
+    id = models.SmallAutoField(primary_key=True)
+    search = models.TextField(unique=True)
+    type = models.TextField(choices=TYPE_ENUM.choices)  # This field type is a guess.
+
+    class Meta:
+        managed = False
+        db_table = 'search'
+
+    def __str__(self):
+        return "[{}: {}]".format(self.type, self.search)
+
+class Source(models.Model):
+    id = models.SmallAutoField(primary_key=True)
+    source = models.TextField(unique=True)
+
+    class Meta:
+        managed = False
+        db_table = 'source'
+
+    def __str__(self):
+        return "[{}]".format(self.source)
+
+class StatusPatternMatching(models.Model):
+    pattern = models.TextField(primary_key=True)
+    priority = models.SmallIntegerField()
+    status = models.TextField()  # This field type is a guess.
+
+    class Meta:
+        managed = False
+        db_table = 'status_pattern_matching'
+
+    def __str__(self):
+        return "{} -> {} [Priority: {}]".format(self.pattern, self.status, self.priority)
+
+class UrlContent(models.Model):
+    id_url = models.OneToOneField('Urls', models.DO_NOTHING, db_column='id_url', primary_key=True)
+    date_published = models.DateTimeField(blank=True, null=True)
+    title = models.TextField(blank=True, null=True)
+    description = models.TextField(blank=True, null=True)
+    content = models.TextField(blank=True, null=True)
+    valid_content = models.BooleanField(blank=True, null=True)
+    language = models.CharField(max_length=2, blank=True, null=True)
+    keywords = ArrayField(models.TextField(blank=True, null=True))  # This field type is a guess.
+    tags = ArrayField(models.TextField(blank=True, null=True))  # This field type is a guess.
+    authors = ArrayField(models.TextField(blank=True, null=True))  # This field type is a guess.
+    image_main_url = models.TextField(blank=True, null=True)
+    images_url = ArrayField(models.TextField(blank=True, null=True))  # This field type is a guess.
+    videos_url = ArrayField(models.TextField(blank=True, null=True))  # This field type is a guess.
+    url_host = models.TextField(blank=True, null=True)
+    site_name = models.TextField(blank=True, null=True)
+
+    class Meta:
+        managed = False
+        db_table = 'url_content'
+
+class Urls(models.Model):
+    class STATUS_ENUM(models.TextChoices):
+        RAW = "raw", "Raw"
+        ERROR = "error", "Error"
+        VALID = "valid", "Valid"
+        UNKNOWN = "unknown", "Unknown"
+        INVALID = "invalid", "Invalid"
+        DUPLICATE = "duplicate", "Duplicate"
+
+    url = models.TextField(unique=True)
+    ts_fetch = models.DateTimeField(auto_now_add=True)
+    status = models.TextField(choices=STATUS_ENUM.choices, default=STATUS_ENUM.RAW)  # This field type is a guess.
+
+    class Meta:
+        managed = False
+        db_table = 'urls'
+        ordering = ["-ts_fetch"]
+
+    def __str__(self):
+        return "URL: {} Fetch:{} Status:{}".format(self.url, self.ts_fetch, self.status)
+
+
+class UrlsDuplicate(models.Model):
+    id_url_canonical = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url_canonical', primary_key=True)  # The composite primary key (id_url_canonical, id_url_duplicated) found, that is not supported. The first column is selected.
+    id_url_duplicated = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
+
+    class Meta:
+        managed = False
+        db_table = 'urls_duplicate'
+        unique_together = (('id_url_canonical', 'id_url_duplicated'),)
+
+    def __str__(self):
+        return "{} {} ".format(self.id_url_duplicated, self.id_url_canonical)
+
+class UrlsSourceSearch(models.Model):
+    id_url = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url', primary_key=True)  # The composite primary key (id_url, id_source, id_search) found, that is not supported. The first column is selected.
+    id_source = models.ForeignKey(Source, models.DO_NOTHING, db_column='id_source')
+    id_search = models.ForeignKey(Search, models.DO_NOTHING, db_column='id_search')
+
+    class Meta:
+        managed = False
+        db_table = 'urls_source_search'
+        unique_together = (('id_url', 'id_source', 'id_search'),)
+
+    def __str__(self):
+        return "{} {} {}".format(self.id_source, self.id_search, self.id_url)
+
+""" # TODO: Migrate to django 5.2
+class UrlsDuplicate(models.Model):
+    pk = models.CompositePrimaryKey('id_url_canonical', 'id_url_duplicated')
+    id_url_canonical = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_canonical')
+    id_url_duplicated = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
+
+    class Meta:
+        managed = False
+        db_table = 'urls_duplicate'
+        unique_together = (('id_url_canonical', 'id_url_duplicated'),)
+
+    def __str__(self):
+        return "{} {} ".format(self.id_url_duplicated, self.id_url_canonical)
+
+class UrlsSourceSearch(models.Model):
+    pk = models.CompositePrimaryKey('id_url', 'id_source', 'id_search')
+    id_url = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url')
+    id_source = models.ForeignKey(Source, models.DO_NOTHING, db_column='id_source')
+    id_search = models.ForeignKey(Search, models.DO_NOTHING, db_column='id_search')
+
+    class Meta:
+        managed = False
+        db_table = 'urls_source_search'
+        unique_together = (('id_url', 'id_source', 'id_search'),)
+
+    def __str__(self):
+        return "{} {} {}".format(self.id_source, self.id_search, self.id_url)
+"""