141 lines
5.8 KiB
Python
141 lines
5.8 KiB
Python
from django.db import models
|
|
from django.contrib.postgres.fields import ArrayField
|
|
|
|
# Create your models here.
|
|
class Search(models.Model):
|
|
class TYPE_ENUM(models.TextChoices):
|
|
RSS_FEED = "rss_feed", "RSS_Feed"
|
|
KEYWORD_SEARCH = "keyword_search", "Keyword_Search"
|
|
URL_HOST = "url_host", "URL_Host"
|
|
|
|
id = models.SmallAutoField(primary_key=True)
|
|
search = models.TextField(unique=True)
|
|
type = models.TextField(choices=TYPE_ENUM.choices) # This field type is a guess.
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'search'
|
|
|
|
def __str__(self):
|
|
return "[{}: {}]".format(self.type, self.search)
|
|
|
|
class Source(models.Model):
|
|
id = models.SmallAutoField(primary_key=True)
|
|
source = models.TextField(unique=True)
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'source'
|
|
|
|
def __str__(self):
|
|
return "[{}]".format(self.source)
|
|
|
|
class StatusPatternMatching(models.Model):
|
|
pattern = models.TextField(primary_key=True)
|
|
priority = models.SmallIntegerField()
|
|
status = models.TextField() # This field type is a guess.
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'status_pattern_matching'
|
|
|
|
def __str__(self):
|
|
return "{} -> {} [Priority: {}]".format(self.pattern, self.status, self.priority)
|
|
|
|
class UrlContent(models.Model):
|
|
id_url = models.OneToOneField('Urls', models.CASCADE, db_column='id_url', primary_key=True)
|
|
date_published = models.DateTimeField(blank=True, null=True)
|
|
title = models.TextField(blank=True, null=True)
|
|
description = models.TextField(blank=True, null=True)
|
|
content = models.TextField(blank=True, null=True)
|
|
valid_content = models.BooleanField(blank=True, null=True)
|
|
language = models.CharField(max_length=2, blank=True, null=True)
|
|
keywords = ArrayField(models.TextField(blank=True, null=True)) # This field type is a guess.
|
|
tags = ArrayField(models.TextField(blank=True, null=True)) # This field type is a guess.
|
|
authors = ArrayField(models.TextField(blank=True, null=True)) # This field type is a guess.
|
|
image_main_url = models.TextField(blank=True, null=True)
|
|
images_url = ArrayField(models.TextField(blank=True, null=True)) # This field type is a guess.
|
|
videos_url = ArrayField(models.TextField(blank=True, null=True)) # This field type is a guess.
|
|
url_host = models.TextField(blank=True, null=True)
|
|
site_name = models.TextField(blank=True, null=True)
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'url_content'
|
|
|
|
class Urls(models.Model):
|
|
class STATUS_ENUM(models.TextChoices):
|
|
RAW = "raw", "Raw"
|
|
ERROR = "error", "Error"
|
|
VALID = "valid", "Valid"
|
|
UNKNOWN = "unknown", "Unknown"
|
|
INVALID = "invalid", "Invalid"
|
|
DUPLICATE = "duplicate", "Duplicate"
|
|
|
|
url = models.TextField(unique=True)
|
|
ts_fetch = models.DateTimeField(auto_now_add=True)
|
|
status = models.TextField(choices=STATUS_ENUM.choices, default=STATUS_ENUM.RAW) # This field type is a guess.
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'urls'
|
|
ordering = ["-ts_fetch"]
|
|
|
|
def __str__(self):
|
|
return "URL: {} Fetch:{} Status:{}".format(self.url, self.ts_fetch, self.status)
|
|
|
|
|
|
class UrlsDuplicate(models.Model):
|
|
id_url_canonical = models.OneToOneField(Urls, models.CASCADE, db_column='id_url_canonical', primary_key=True) # The composite primary key (id_url_canonical, id_url_duplicated) found, that is not supported. The first column is selected.
|
|
id_url_duplicated = models.ForeignKey(Urls, models.CASCADE, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'urls_duplicate'
|
|
unique_together = (('id_url_canonical', 'id_url_duplicated'),)
|
|
|
|
def __str__(self):
|
|
return "{} {} ".format(self.id_url_duplicated, self.id_url_canonical)
|
|
|
|
class UrlsSourceSearch(models.Model):
|
|
id_url = models.OneToOneField(Urls, models.CASCADE, db_column='id_url', primary_key=True) # The composite primary key (id_url, id_source, id_search) found, that is not supported. The first column is selected.
|
|
#id_url = models.ForeignKey(Urls, models.CASCADE, db_column='id_url')
|
|
id_source = models.ForeignKey(Source, models.CASCADE, db_column='id_source')
|
|
id_search = models.ForeignKey(Search, models.CASCADE, db_column='id_search')
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'urls_source_search'
|
|
unique_together = (('id_url', 'id_source', 'id_search'),)
|
|
|
|
def __str__(self):
|
|
return "{} {} {}".format(self.id_source, self.id_search, self.id_url)
|
|
|
|
""" # TODO: Migrate to django 5.2
|
|
class UrlsDuplicate(models.Model):
|
|
pk = models.CompositePrimaryKey('id_url_canonical', 'id_url_duplicated')
|
|
id_url_canonical = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_canonical')
|
|
id_url_duplicated = models.ForeignKey(Urls, models.DO_NOTHING, db_column='id_url_duplicated', related_name='urlsduplicate_id_url_duplicated_set')
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'urls_duplicate'
|
|
unique_together = (('id_url_canonical', 'id_url_duplicated'),)
|
|
|
|
def __str__(self):
|
|
return "{} {} ".format(self.id_url_duplicated, self.id_url_canonical)
|
|
|
|
class UrlsSourceSearch(models.Model):
|
|
pk = models.CompositePrimaryKey('id_url', 'id_source', 'id_search')
|
|
id_url = models.OneToOneField(Urls, models.DO_NOTHING, db_column='id_url')
|
|
id_source = models.ForeignKey(Source, models.DO_NOTHING, db_column='id_source')
|
|
id_search = models.ForeignKey(Search, models.DO_NOTHING, db_column='id_search')
|
|
|
|
class Meta:
|
|
managed = False
|
|
db_table = 'urls_source_search'
|
|
unique_together = (('id_url', 'id_source', 'id_search'),)
|
|
|
|
def __str__(self):
|
|
return "{} {} {}".format(self.id_source, self.id_search, self.id_url)
|
|
""" |