matitos/matitos_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Towards django RQ

Browse Source
This commit is contained in:
Luciano Gervasoni
2025-03-10 12:17:31 +01:00
parent e024b200bb
commit e124dbc21a
20 changed files with 722 additions and 4643 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
A_Development.ipynb
View File

@@ -103,13 +103,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 54,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"('https://foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert',\n",
" 'foxnews.com')"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# !pip install trafilatura trafilatura[all] cchardet\n",
"import courlan\n",
"url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
"url = \"https://foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
"courlan.check_url(url)"
]
},
@@ -125,13 +138,65 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [],
"source": [
"import newspaper\n",
"\n",
"article = newspaper.article(url)"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"datetime.datetime(2025, 3, 4, 4, 0, 31, tzinfo=tzoffset(None, -18000))"
]
},
"execution_count": 49,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"article.publish_date"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"# !pip install trafilatura\n",
"import trafilatura\n",
"from pprint import pprint\n",
"\n",
"url = \"https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert\"\n",
"url = \"https://www.missingkids.org/poster/USVA/VA25-0820/1\"\n",
"# url = \"https://www.missingkids.org/poster/USVA/VA25-0820/1\"\n",
"\n",
"# Fetch\n",
"doc = trafilatura.fetch_url(url)\n",
@@ -142,9 +207,40 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'author': 'Audrey Conklin',\n",
" 'body': <Element body at 0x7e22813ce400>,\n",
" 'categories': [],\n",
" 'comments': None,\n",
" 'commentsbody': <Element body at 0x7e22813ce180>,\n",
" 'date': '2025-03-03',\n",
" 'description': \"Disgraced parenting blogger and mom of six Ruby Franke's \"\n",
" '\"power\" and public image\" allowed her crimes against her '\n",
" 'children to go \"unchecked,\" according to a defense attorney.',\n",
" 'filedate': '2025-03-08',\n",
" 'fingerprint': None,\n",
" 'hostname': 'foxnews.com',\n",
" 'id': None,\n",
" 'image': 'https://static.foxnews.com/foxnews.com/content/uploads/2024/03/967e1c1b-Franke.jpg',\n",
" 'language': None,\n",
" 'license': None,\n",
" 'pagetype': 'article',\n",
" 'raw_text': None,\n",
" 'sitename': 'Fox News',\n",
" 'tags': [],\n",
" 'text': None,\n",
" 'title': \"Utah mommy blogger Ruby Franke's power, public image allowed child \"\n",
" \"abuse to go 'unchecked': expert\",\n",
" 'url': 'https://www.foxnews.com/us/utah-mommy-blogger-ruby-franke-power-public-image-allowed-child-abuse-go-unchecked-expert'}\n"
]
}
],
"source": [
"pprint(metadata.as_dict())"
]
@@ -165,6 +261,64 @@
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 18.6 ms, sys: 40 μs, total: 18.7 ms\n",
"Wall time: 18 ms\n"
]
},
{
"data": {
"text/plain": [
"'en'"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"'''\n",
"!pip install lingua-language-detector\n",
"import lingua\n",
"ld = lingua.LanguageDetectorBuilder.from_all_languages().build()\n",
"l = ld.detect_language_of(content)\n",
"'''\n",
"# !pip install langdetect \n",
"import langdetect\n",
"langdetect.DetectorFactory.seed = 0\n",
"langdetect.detect(content)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,