matitos_news/utils/Newspapers.ipynb

In [ ]:

url = "https://onlinenewspapers.com/index.shtml"

In [2]:

"""
import newspaper

newspaper.Config().__dict__

 'requests_params': {'timeout': 7,
  'proxies': {},
  'headers': {'User-Agent': 'newspaper/0.9.3.1'}},
"""
import newspaper
newspaper.Config().browser_user_agent

Out [2]:

'newspaper/0.9.3.1'

In [ ]:

"""
        url (str): The url of the source (news website) to build. For example,
            `https://www.cnn.com`.
        dry (bool): If true, the source object will be constructed but not
            downloaded or parsed.
        only_homepage (bool): If true, the source object will only parse
            the homepage of the source.
        only_in_path (bool): If true, the source object will only
            parse the articles that are in the same path as the source's
            homepage. You can scrape a specific category this way.
            Defaults to False.
        input_html (str): The HTML of the source to parse. Use this to pass cached
            HTML to the source object.
        config (Configuration): A configuration object to use for the source.
        kwargs: Any other keyword arguments to pass to the Source constructor.
            If you omit the config object, you can add any configuration
            options here.
"""

url = "https://www.lanacion.com.ar/deportes/"

newspaper_built = newspaper.build(url, only_in_path=True)

In [ ]:

newspaper_built.__dict__

In [ ]:

newspaper_built.article_urls()

In [ ]:

url = "https://www.lanacion.com.ar/"
#url = "https://www.lanacion.com.ar/deportes/"
newspaper_built = newspaper.build(url)

In [ ]:

"""
        url (str): The url of the source (news website) to build. For example,
            `https://www.cnn.com`.
        dry (bool): If true, the source object will be constructed but not
            downloaded or parsed.
        only_homepage (bool): If true, the source object will only parse
            the homepage of the source.
        only_in_path (bool): If true, the source object will only
            parse the articles that are in the same path as the source's
            homepage. You can scrape a specific category this way.
            Defaults to False.
        input_html (str): The HTML of the source to parse. Use this to pass cached
            HTML to the source object.
        config (Configuration): A configuration object to use for the source.
        kwargs: Any other keyword arguments to pass to the Source constructor.
            If you omit the config object, you can add any configuration
            options here.
"""

In [ ]:

cat = newspaper_built.categories[0]

In [ ]:

newspaper_built.categories_to_articles()

In [ ]:

newspaper_built.category_urls()

In [ ]:

 'https://www.lanacion.com.ar/tema/futbol-argentino-tid57505/',

In [ ]:

categories = newspaper_built.category_urls()
url_of_interest = "https://www.lanacion.com.ar/sabado/todo-para-ellos-nid21042025/"

potential_categories = []

for c in categories:
    if (c in url_of_interest):
        print(c, url_of_interest)
        potential_categories.append(c)

# Get longest length category

In [ ]:

newspaper_built.article_urls()