from newspaper import Config

config = Config()
config.browser_user_agent = string value
config.proxies = dictionary of proxies
config.request_timeout = int value 

from newspaper import Config

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

# add your proxy information
PROXIES = {
           'http': "http://ip_address:port_number",
           'https': "https://ip_address:port_number"
          }

config = Config()
config.browser_user_agent = USER_AGENT
config.proxies = PROXIES
config.request_timeout = 10

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com'
article = Article(base_url, config=config)
 <DO SOMETHING>

from newspaper import Config
from newspaper import Article

HEADERS = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}
           
config = Config()
config.headers = HEADERS
config.request_timeout = 10

base_url = 'https://www.wsj.com'
article = Article(base_url, config=config)
 <DO SOMETHING>

import json
from newspaper import Config
from newspaper import Article
from newspaper.utils import BeautifulSoup

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.bbc.com/news/health-54500673'
article = Article(base_url, config=config)
article.download()
article.parse()

print(article.title)
['Covid virus ‘survives for 28 days’ in lab conditions']

article_meta_data = article.meta_data

article_summary = {value for (key, value) in article_meta_data.items() if key == 'description'}
print(article_summary)
{'Researchers find SARS-Cov-2 survives for longer than thought - but only under certain conditions.'}

soup = BeautifulSoup(article.html, 'html.parser')
bbc_dictionary = json.loads("".join(soup.find("script", {"type":"application/ld+json"}).contents))

date_published = [value for (key, value) in bbc_dictionary.items() if key == 'datePublished']
print(date_published)
['2020-10-11T20:11:33.000Z']

article_author = [value['name'] for (key, value) in bbc_dictionary.items() if key == 'author']
print(article_author)
['BBC News']

# another method to extract the title
article_title = [value for (key, value) in bbc_dictionary.items() if key == 'headline']
print(article_title)
['Covid virus ‘survives for 28 days’ in lab conditions']

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

url = 'https://www.cnn.com/2020/10/09/business/edinburgh-woollen-mill-job-cuts/index.html'
article = Article(url, config=config)
article.download()
article.parse()

print(article.title)
Another 24,000 retail jobs at risk as UK fashion group faces collapse

print(article.publish_date)
2020-10-09 00:00:00

print(article.authors)
['Hanna Ziady', 'Cnn Business']

print(article.keywords)
[] returned an empty list

print(article.meta_keywords)
['business', 'Edinburgh Woollen Mill: 24', '000 jobs at risk as company appoints administrators - CNN']

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.foxbusiness.com/economy/white-house-calls-for-interim-coronavirus-relief-as-negotiations-continue'
article = Article(url, config=config)
article.download()
article.parse()
article_meta_data = article.meta_data

article_title = {value for (key, value) in article_meta_data.items() if key == 'dc.title'}
print(article_title)
{'White House pushes for limited coronavirus relief bill as broader effort meets resistance'}

article_published_date = str({value for (key, value) in article_meta_data.items() if key == 'dcterms.created'})
print(article_published_date)
{'2020-10-11T12:51:53-04:00'}

article_summary = {value for (key, value) in article_meta_data.items() if key == 'dc.description'}
print(article_summary)
{'In the letter to House and Senate members, Mnuchin and Meadows said the White House would continue to talk to Senate Democratic Leader Chuck Schumer 
and House Speaker Nancy Pelosi, but that Congress should "immediately vote on a bill" that would enable the use of unused Paycheck Protection Program 
funds while working toward a bigger package.'}

import json
from newspaper import Config
from newspaper import Article
from newspaper.utils import BeautifulSoup

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.foxbusiness.com/economy/white-house-calls-for-interim-coronavirus-relief-as-negotiations-continue'
article = Article(base_url, config=config)
article.download()
article.parse()

soup = BeautifulSoup(article.html, 'html.parser')
cnn_dictionary = json.loads("".join(soup.find("script", {"type":"application/ld+json"}).contents))

date_published = [value for (key, value) in cnn_dictionary.items() if key == 'datePublished']
print(date_published)
['2020-10-11T12:51:53-04:00']

article_author = [value['name'] for (key, value) in cnn_dictionary.items() if key == 'author']
print(article_author)
['Reuters']

article_title = [value for (key, value) in cnn_dictionary.items() if key == 'headline']
print(article_title)
['White House pushes for limited coronavirus relief bill as broader effort meets resistance']

article_summary = [value for (key, value) in cnn_dictionary.items() if key == 'description']
print(article_summary)
['In the letter to House and Senate members, Mnuchin and Meadows said the White House would continue to talk to Senate Democratic Leader Chuck Schumer 
and House Speaker Nancy Pelosi, but that Congress should "immediately vote on a bill" that would enable the use of unused Paycheck Protection Program 
funds while working toward a bigger package.']

import json
import requests
import pandas as pd
from newspaper import Config
from newspaper import Article
from newspaper.utils import BeautifulSoup

HEADERS = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}


def query_foxbaltimore_news():
    df_foxbaltimore_extraction = pd.DataFrame(columns=['article_category', 'date_published', 'article authors',
                                                       'article title', 'article summary', 'article keywords',
                                                       'article url', 'article text'])

    url = 'http://foxbaltimore.com/'
    response = requests.get(url, headers=HEADERS, allow_redirects=True, verify=True, timeout=30)
    soup = BeautifulSoup(response.content, 'html.parser')
    fox_soup = soup.find_all("script", {"type": "application/json"})[1]
    fox_json = json.loads(''.join(fox_soup))
    for news in fox_json['content']['page-data']['teaser']:
        for article in news['teasers']:
            article_category = article['categories'][0]
            article_title = article['title']
            article_url = f"https://foxbaltimore.com{article['url']}"
            article_summary = article['summary']
            article_published_date = article['publishedDateISO8601']
            if 'sponsored' not in article_url:
                article_details = query_individual_article_elements(article_url)
                df_foxbaltimore_extraction = df_foxbaltimore_extraction.append({'article category':article_category,
                                                                       'date_published': article_published_date,
                                                                       'article authors': article_details[0],
                                                                       'article title': article_title,
                                                                       'article summary': article_summary,
                                                                       'article keywords': article_details[3],
                                                                       'article url': article_url,
                                                                       'article text': article_details[5]}, ignore_index=True)
    return df_foxbaltimore_extraction


def query_individual_article_elements(url):
    config = Config()
    config.headers = HEADERS
    config.request_timeout = 30
    article = Article(url, config=config, memoize_articles=False)
    article.download()
    article.parse()
    article_meta_data = article.meta_data

    article_author = article.authors

    article_published_date = str({value['published_time'] for (key, value) in article_meta_data.items()
                                  if key == 'article'})

    article_keywords = sorted([value.lower() for (key, value) in article_meta_data.items() if key == 'keywords'])

    article_title = str({value for (key, value) in article_meta_data.items() if key == 'title'})

    article_summary = {value for (key, value) in article_meta_data.items() if key == 'description'}

    soup = BeautifulSoup(article.html, 'html.parser')
    fox_soup = soup.find_all("script", {"type": "application/json"})[1]
    fox_json = json.loads(''.join(fox_soup))
    article_text = ''.join(fox_json['content']['main_content']['story']['richText'])
    article_details = [article_author,
                       article_published_date,
                       article_title,
                       article_keywords,
                       article_summary,
                       article_text]

    return article_details

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1'
article = Article(base_url, config=config)
article.download()
article.parse()

print(article.title)
Investors Are Betting Corporate Earnings Have Turned a Corner

print(article.authors)
['Karen Langley']

print(article.publish_date)
None

print(article.keywords)
[] returned an empty list

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1'
article = Article(base_url, config=config)
article.download()
article.parse()
article_meta_data = article.meta_data

article_published_date = str({value for (key, value) in article_meta_data.items() if key == 'article.published'})
print(article_published_date)
{'2020-10-11T09:30:00.000Z'}

article_author = sorted({value for (key, value) in article_meta_data.items()if key == 'author'})
print(article_author)
['Karen Langley']

article_title = {value for (key, value) in article_meta_data.items() if key == 'article.headline'}
print(article_title)
{'Investors Are Betting Corporate Earnings Have Turned a Corner'}

article_summary = {value for (key, value) in article_meta_data.items() if key == 'article.summary'}
print(article_summary)
{'Investors are entering third-quarter earnings season with brighter expectations for corporate profits, 
a bet they hope will propel the next leg of the stock market’s rally.'}

keywords = ''.join({value for (key, value) in article_meta_data.items() if key == 'news_keywords'})
article_keywords = sorted(keywords.lower().split(','))
print(article_keywords)
['c&e exclusion filter', 'c&e industry news filter', 'codes_reviewed', 'commodity/financial market news', 'content types', 
'corporate/industrial news', 'earnings', 'equity markets', 'factiva filters', 'financial performance']

from time import sleep
from random import randint
from newspaper import Config
from newspaper import Source

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

cnbc_wayback_archive = Source(url='https://web.archive.org/web/20180301012621/https://www.cnbc.com/', config=config,
                      memoize_articles=False, language='en', number_threads=20, thread_timeout_seconds=2)

cnbc_wayback_archive.build()
for article in cnbc_wayback_archive.articles:
    article.download()
    article.parse()
    article_meta_data = article.meta_data

    print(article.publish_date)
    print(article.title)

    article_description = "".join({value for (key, value) in article_meta_data.items() if key == 'description'})
    print(article_description)

    article_keywords = {value for (key, value) in article_meta_data.items() if key == 'keywords'}
    print(list(article_keywords))

    print(article.url)

    # this sleep timer is helping with some timeout issues
    # that happened when querying
    sleep(randint(1, 5))

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.cnn.com/2020/10/12/health/johnson-coronavirus-vaccine-pause-bn/index.html'
article = Article(base_url, config=config)
article.download()
article.parse()
with open('cnn.html', 'w') as fileout:
    fileout.write(article.html)


# Read the HTML file created above
with open("cnn.html", 'r') as f:
    # note the empty URL string
    article = Article('', language='en')
    article.download(input_html=f.read())
    article.parse()
    
    print(article.title)
    Johnson & Johnson pauses Covid-19 vaccine trial after 'unexplained illness'
    
    article_meta_data = article.meta_data
    
    article_published_date = {value for (key, value) in article_meta_data.items() if key == 'pubdate'}
    print(article_published_date)
    {'2020-10-13T01:31:25Z'}

    article_author = {value for (key, value) in article_meta_data.items() if key == 'author'}
    print(article_author)
    {'Maggie Fox, CNN'}

    article_summary = {value for (key, value) in article_meta_data.items() if key == 'description'}
    print(article_summary)
    {'Johnson&Johnson said its Janssen arm had paused its coronavirus vaccine trial  after an "unexplained illness" in one 
    of the volunteers testing its experimental Covid-19 shot.'}

    article_keywords = {value for (key, value) in article_meta_data.items() if key == 'keywords'}
    print(article_keywords)
    {"health, Johnson & Johnson pauses Covid-19 vaccine trial after 'unexplained illness' - CNN"}

article_meta_data = article.meta_data
article_author = {value for (key, value) in article_meta_data['article'].items() if key == 'author'}

from newspaper.utils import BeautifulSoup

article = Article(website, config=config)
article.download()
article.parse()

soup = BeautifulSoup(article.html, 'html.parser')
la_times_dictionary = json.loads("".join(soup.find("script", {"type": "application/ld+json"}).contents))
article_author = ''.join([value[0]['name'] for (key, value) in la_times_dictionary.items() if key == 'author'])

cnn_articles = newspaper.build('https://www.cnn.com/', config=config)

article = Article('https://www.cnn.com/2020/12/05/health/us-hospitals-covid-pandemic/index.html', config=config)

cnn_articles = newspaper.build('https://www.cnn.com/', config=config,  memoize_articles=False)

article = Article('https://www.cnn.com/2020/12/05/health/us-hospitals-covid-pandemic/index.html', config=config, memoize_articles=False)

import newspaper
newspaper.languages()

Your available languages are:
input code      full name

  ar              Arabic
  be              Belarusian
  bg              Bulgarian
  da              Danish
  de              German
  el              Greek
  en              English
  es              Spanish
  et              Estonian
  fa              Persian
  fi              Finnish
  fr              French
  he              Hebrew
  hi              Hindi
  hr              Croatian
  hu              Hungarian
  id              Indonesian
  it              Italian
  ja              Japanese
  ko              Korean
  lt              Lithuanian
  mk              Macedonian
  nb              Norwegian (Bokmål)
  nl              Dutch
  no              Norwegian
  pl              Polish
  pt              Portuguese
  ro              Romanian
  ru              Russian
  sl              Slovenian
  sr              Serbian
  sv              Swedish
  sw              Swahili
  th              Thai
  tr              Turkish
  uk              Ukrainian
  vi              Vietnamese
  zh              Chinese

from newspaper import Config
from newspaper import Article
import jieba
import logging
jieba.setLogLevel(logging.ERROR)

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'http://tech.chinadaily.com.cn/a/202009/30/WS5f7414f1a3101e7ce9727a44.html'
article = Article(base_url, config=config, language='zh')
article.download()
article.parse()
article_meta_data = article.meta_data

print(article.title)
中国发布高分多模卫星首批影像成果

print(article.publish_date)
2020-09-30 00:00:00

article_keywords = {value for (key, value) in article_meta_data.items() if key == 'Keywords'}
if article_keywords:
    print(article_keywords)
    {'多模,高分,影像,卫星,成果,发布,中国'}

import json
import requests
from newspaper import Article
from newspaper.utils import BeautifulSoup

HEADERS = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}

base_url = 'https://www.zeit.de/politik/ausland/2020-10/us-wahl-donald-trump-gewalt-milizen-protest'
raw_html = requests.get(base_url, headers=HEADERS, timeout=10)
article = Article('', language='de')
article.download(input_html=raw_html.content)
article.parse()

soup = BeautifulSoup(article.html, 'html.parser')
zeit_dictionary = json.loads("".join(soup.findAll("script", {"type": "application/ld+json"})[3].contents))

date_published = [value for (key, value) in zeit_dictionary.items() if key == 'datePublished']
print(date_published)
['2020-10-12T04:53:14+02:00']

article_author = [value['name'] for (key, value) in zeit_dictionary.items() if key == 'author']
print(article_author)
['Rieke Havertz']

article_title = [value for (key, value) in zeit_dictionary.items() if key == 'headline']
print(article_title)
['US-Wahl: Gewalt nicht ausgeschlossen']

article_summary = [value for (key, value) in zeit_dictionary.items() if key == 'description']
print(article_summary)
['Tote bei Protesten zwischen Linken und Rechten, Terrorpläne im eigenen Land: Die Gewaltbereitschaft in den USA ist vor der Wahl hoch. Und der Präsident deeskaliert nicht.']

import sys

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import WebDriverException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains

from bs4 import BeautifulSoup

from newspaper import Article
from newspaper import Config

# config details for newspaper
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'
config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10


def get_chrome_webdriver():
    chrome_options = Options()
    chrome_options.add_argument("--test-type")
    chrome_options.add_argument('--ignore-certificate-errors')
    chrome_options.add_argument('--disable-extensions')
    chrome_options.add_argument('disable-infobars')
    chrome_options.add_argument("--incognito")
    # chrome_options.add_argument('--headless')

    # window size as an argument is required in headless mode
    chrome_options.add_argument('window-size=1920x1080')
    
    # disable the banner "Chrome is being controlled by automated test software"
    chrome_options.add_experimental_option("useAutomationExtension", False)
    chrome_options.add_experimental_option("excludeSwitches", ['enable-automation'])
    
    driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
    return driver


def get_chrome_browser(url):
    browser = get_chrome_webdriver()
    browser.get(url)
    return browser


def chrome_browser_teardown(browser):
    browser.close()
    browser.quit()
    return


def bypass_popup_warnings(browser):
    try:
        hidden_element = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.ID, "wzrk-cancel")))
        if hidden_element.is_displayed():
            browser.implicitly_wait(20)
            subscribe_button = browser.find_element_by_xpath("//*[@id='wzrk-cancel']")
            ActionChains(browser).move_to_element(subscribe_button).click(subscribe_button).perform()
            browser.implicitly_wait(20)
            cookie_button = browser.find_element_by_xpath("//span[@onclick='createCookie()']")
            ActionChains(browser).move_to_element(cookie_button).click(cookie_button).perform()
            return True
        else:
            browser.implicitly_wait(20)
            cookie_button = browser.find_element_by_xpath("//span[@onclick='createCookie()']")
            ActionChains(browser).move_to_element(cookie_button).click(cookie_button).perform()
            return True

    except NoSuchElementException:
        print('Webdriver is unable to identify the requested element during runtime.')
        sys.exit(1)

    except WebDriverException:
        print('The Element Click command could not be completed because the element receiving the events is obscuring the element that was requested clicked.')
        sys.exit(1)


def query_al_arabiya_news(browser):
    news_urls = []
    soup = BeautifulSoup(browser.page_source, 'lxml')
    for a in soup.find_all('a', href=True):
        if str(a['href']).startswith('/ar/'):
            news_urls.append(f"https://www.alarabiya.net/{a['href']}")
    for url in news_urls:
        article = Article(url, config=config, language='ar')
        article.download()
        article.parse()
        
        # additional code required to extract article elements
        # please review the page source to determine the techniques 
        # needed
        print(article.title)
        
    return True


news_browser = get_chrome_browser('https://www.alarabiya.net')
warnings_closed = bypass_popup_warnings(news_browser)
if warnings_closed is True:
    finished = query_al_arabiya_news(news_browser)
    if finished is True:
        chrome_browser_teardown(news_browser)

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException

from bs4 import BeautifulSoup

from newspaper import Article
from newspaper import Config

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'
config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10


def get_chrome_webdriver():
    chrome_options = Options()
    chrome_options.add_argument("--test-type")
    chrome_options.add_argument('--ignore-certificate-errors')
    chrome_options.add_argument('--disable-extensions')
    chrome_options.add_argument('disable-infobars')
    chrome_options.add_argument("--incognito")
    # chrome_options.add_argument('--headless')

    # window size as an argument is required in headless mode
    chrome_options.add_argument('window-size=1920x1080')
    
    # disable the banner "Chrome is being controlled by automated test software"
    chrome_options.add_experimental_option("useAutomationExtension", False)
    chrome_options.add_experimental_option("excludeSwitches", ['enable-automation'])
    
    driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)
    return driver


def get_chrome_browser(url):
    browser = get_chrome_webdriver()
    browser.get(url)
    return browser


def chrome_browser_teardown(browser):
    browser.close()
    browser.quit()
    return


def bypass_gdpr_acknowledgement(browser):
    if browser.find_elements_by_tag_name('iframe'):
        iframes = browser.find_elements_by_tag_name('iframe')
        number_of_iframes = len(iframes)
        for i in range(number_of_iframes):
            browser.switch_to.frame(i)
            if browser.find_elements_by_tag_name("title"):
                title_element = browser.find_element_by_tag_name("title").get_attribute("innerHTML")
                if title_element == "Notice Message App":
                    warning_labels = ['Akzeptieren', 'Alle akzeptieren', 'ZUSTIMMEN']
                    for label in warning_labels:
                        try:
                            browser.find_element_by_xpath(f'//button[text()="{label}"]').click()
                            browser.switch_to.default_content()
                            browser.implicitly_wait(10)
                            return True
                        except NoSuchElementException:
                            pass
                else:
                    browser.switch_to.default_content()
            else:
                browser.switch_to.default_content()
    else:
        return False


def query_german_news_site(browser):
    """
    This function needs to be configured to harvest from the site that 
    is being queried. Please reference the Al Arabiya Extraction in Arabic example
    for guidance. 
    """
    soup = BeautifulSoup(browser.page_source, 'lxml')
    for a in soup.find_all('a', href=True):
        print(a['href'])
    return True


news_browser = get_chrome_browser('https://www.handelsblatt.com/')
gdpr_status = bypass_gdpr_acknowledgement(news_browser)
if gdpr_status is True:
    finished = query_german_news_site(news_browser)
    if finished is True:
        chrome_browser_teardown(news_browser)

import csv
from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1'
article = Article(base_url, config=config)
article.download()
article.parse()
article_meta_data = article.meta_data

published_date = {value for (key, value) in article_meta_data.items() if key == 'article.published'}
article_published_date = " ".join(str(x) for x in published_date)

authors = sorted({value for (key, value) in article_meta_data.items()if key == 'author'})
article_author = ', '.join(authors)

title = {value for (key, value) in article_meta_data.items() if key == 'article.headline'}
article_title = " ".join(str(x) for x in title)

summary = {value for (key, value) in article_meta_data.items() if key == 'article.summary'}
article_summary = " ".join(str(x) for x in summary)

keywords = ''.join({value for (key, value) in article_meta_data.items() if key == 'news_keywords'})
keywords_list = sorted(keywords.lower().split(','))
article_keywords = ', '.join(keywords_list)

with open('wsj_extraction_results.csv', 'a', newline='') as csvfile:
    headers = ['date published', 'article authors', 'article title', 'article summary', 'article keywords']
    writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n', fieldnames=headers)
    writer.writeheader()

    writer.writerow({'date published': article_published_date,
                     'article authors': article_author,
                     'article title': article_title,
                     'article summary': article_summary,
                     'article keywords': article_keywords})

import json
import pandas as pd
from datetime import datetime
from newspaper import Config
from newspaper import Article
from newspaper.utils import BeautifulSoup

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10


def path_to_image_html(link):
    """
    Converts image links to HTML tags
    :param link: image URL
    :return: URL wrapped in clickable HTML tag
    """
    return f'<a href="{link}"> <img src="{link}" width="60" > </a>'


def harvest_article_content(website):
    """
    Queries and extracts specific content from a LA Times article.
    :param website: URL for a LA Times article
    :return: pandas dataframe
    """
    df_latimes_extraction = pd.DataFrame(columns=['Date Published', 'URL', 'Author', 'Title',
                                                  'Summary', 'Text', 'Main Image'])

    article = Article(website, config=config)
    article.download()
    article.parse()

    soup = BeautifulSoup(article.html, 'html.parser')
    la_times_dictionary = json.loads("".join(soup.find("script", {"type": "application/ld+json"}).contents))

    date_published = ''.join([value for (key, value) in la_times_dictionary.items() if key == 'datePublished'])
    clean_date = datetime.strptime(date_published, "%Y-%m-%dT%H:%M:%S.%f%z").strftime('%Y-%m-%d')

    article_author = ''.join([value[0]['name'] for (key, value) in la_times_dictionary.items() if key == 'author'])
    article_title = ''.join([value for (key, value) in la_times_dictionary.items() if key == 'headline'])
    article_url = ''.join([value for (key, value) in la_times_dictionary.items() if key == 'url'])
    article_description = ''.join([value for (key, value) in la_times_dictionary.items() if key == 'description'])
    article_body = ''.join([value.replace('\n', ' ') for (key, value) in la_times_dictionary.items() if key ==
                            'articleBody'])

    local_df = save_article_data(df_latimes_extraction, clean_date,
                                 f'<a href="{article_url}">{article_url}</a>',
                                 article_author,
                                 article_title,
                                 article_description,
                                 article_body,
                                 article.top_image)
    return local_df


def save_article_data(df, published_date, website, authors, title, summary, text, main_image):
    """
    Writes extracted article content to a pandas dataframe.

    :param df: pandas dataframe
    :param published_date: article's published date
    :param website: article's URL
    :param authors: article's author
    :param title: article's title
    :param summary: article's summary
    :param text: article's text
    :param main_image: article's top image
    :return: pandas dataframe
    """
    local_df = df.append({'Date Published': published_date,
                          'URL': website,
                          'Author': authors,
                          'Title': title,
                          'Summary': summary,
                          'Text': text,
                          'Main Image': path_to_image_html(main_image)}, ignore_index=True)
    return local_df


def create_html_file(df):
    """
    Writes a pandas dataframe that contains extracted article content to a HTML file.

    :param df: pandas dataframe
    :return:
    """
    pd.set_option('colheader_justify', 'center')

    html_string = '''
    <html>
      <head>
      <meta charset="utf-8">
      <title>Los Angeles Times Article Information</title></head>
      <link rel="stylesheet" type="text/css" href="df_style.css"/>
      <body>
        {table}
      </body>
    </html>.
    '''

    with open('latimes_results.html', 'w') as f:
        f.write(html_string.format(table=df.to_html(index=False, escape=False, classes='mystyle')))

    return None


# List used to store pandas content extracted 
# from articles.
article_data = []

urls = ['https://www.latimes.com/environment/story/2021-02-10/earthquakes-climate-change-threaten-california-dams',
        'https://www.latimes.com/business/story/2021-02-08/tesla-invests-in-bitcoin',
        'https://www.latimes.com/business/story/2021-02-09/joe-biden-wants-100-clean-energy-will-california-show-that-its-possible']

for url in urls:
    results = harvest_article_content(url)
    article_data.append(results)

# concat all the article content into a new pandas dataframe.
df_latimes = pd.concat(article_data)

# Create the HTML file 
create_html_file(df_latimes)

import json
from newspaper import Config
from newspaper import Article

news_extraction_results = {}

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1'
article = Article(base_url, config=config)
article.download()
article.parse()
article_meta_data = article.meta_data

published_date = {value for (key, value) in article_meta_data.items() if key == 'article.published'}
article_published_date = " ".join(str(x) for x in published_date)

authors = sorted({value for (key, value) in article_meta_data.items()if key == 'author'})
article_author = ', '.join(authors)

title = {value for (key, value) in article_meta_data.items() if key == 'article.headline'}
article_title = " ".join(str(x) for x in title)

summary = {value for (key, value) in article_meta_data.items() if key == 'article.summary'}
article_summary = " ".join(str(x) for x in summary)

keywords = ''.join({value for (key, value) in article_meta_data.items() if key == 'news_keywords'})
keywords_list = sorted(keywords.lower().split(','))
article_keywords = ', '.join(keywords_list)

news_extraction_results['wsj'] = []
news_extraction_results['wsj'].append({
    'published_date': article_published_date,
    'authors': article_author,
    'summary': article_summary,
    'keywords': article_keywords,
    'source url': article.url})


# write JSON file
with open('wsj.json', 'w') as json_file:
    json.dump(news_extraction_results, json_file)
    
# read JSON file
with open('wsj.json') as json_file:
    data = json.load(json_file)
    print(json.dumps(data, indent=4))
    {
      "wsj": [
       {
         "published_date": "2020-10-11T09:30:00.000Z",
         "authors": "Karen Langley",
         "summary": "Investors are entering third-quarter earnings season with brighter expectations for corporate profits, a bet they hope 
         will propel the next leg of the stock market\u2019s rally.",
         "keywords": "c&e exclusion filter, c&e industry news filter, codes_reviewed, commodity/financial market news, 
         content types, corporate/industrial news, earnings, equity markets, factiva filters, financial performance",
         "source url": "https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1"
       }
     ]
   }

import pandas as pd
from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

base_url = 'https://www.wsj.com/articles/investors-are-betting-corporate-earnings-have-turned-a-corner-11602408600?mod=hp_lead_pos1'
article = Article(base_url, config=config)
article.download()
article.parse()
article_meta_data = article.meta_data

published_date = {value for (key, value) in article_meta_data.items() if key == 'article.published'}
article_published_date = " ".join(str(x) for x in published_date)

authors = sorted({value for (key, value) in article_meta_data.items()if key == 'author'})
article_author = ', '.join(authors)

title = {value for (key, value) in article_meta_data.items() if key == 'article.headline'}
article_title = " ".join(str(x) for x in title)

summary = {value for (key, value) in article_meta_data.items() if key == 'article.summary'}
article_summary = " ".join(str(x) for x in summary)

keywords = ''.join({value for (key, value) in article_meta_data.items() if key == 'news_keywords'})
keywords_list = sorted(keywords.lower().split(','))
article_keywords = ', '.join(keywords_list)

# pandas DataFrame used to store the extraction results
df_wsj_extraction = pd.DataFrame(columns=['date_published', 'article authors', 'article title',
                                          'article summary', 'article keywords'])

df_wsj_extraction = df_wsj_extraction.append({'date_published': article_published_date,
                                              'article authors': article_author,
                                              'article title': article_title,
                                              'article summary': article_summary,
                                              'article keywords': article_keywords}, ignore_index=True)

print(df_wsj_extraction.to_string(index=False))

import newspaper
from newspaper import Config
from newspaper import news_pool

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

wsj_news = newspaper.build('https://www.wsj.com/', config=config, memoize_articles=False, language='en')
cnn_news = newspaper.build('https://www.cnn.com/', config=config, memoize_articles=False, language='en')
news_sources = [wsj_news, cnn_news]

# the parameters number_threads and thread_timeout_seconds are adjustable
news_pool.config.number_threads = 4
news_pool.config.thread_timeout_seconds = 1
news_pool.set(news_sources)
news_pool.join()

article_urls = set()
for source in news_sources:
    for article_extract in source.articles:
        if article_extract.url not in article_urls:
            article_urls.add(article_extract.url)
            print(article_extract.title)

from newspaper import Config
from newspaper import Source

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

wsj_news = Source(url='https://www.wsj.com/', config=config, memoize_articles=False, language='en',
                  number_threads=20, thread_timeout_seconds=2)

cnn_news = Source(url='https://www.cnn.com', config=config, memoize_articles=False, language='en',
                  number_threads=20, thread_timeout_seconds=2)

news_sites = [cnn_news, wsj_news]
for site in news_sites:
    site.build()
    for article_extract in site.articles:
        article_extract.download()
        article_extract.parse()
        print(article_extract.title)

from newspaper import Config
from newspaper import Article

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

url = 'https://www.newsweek.com/facebook-super-spreader-election-misinformation-1543306'
article = Article(url, config=config)
article.download()
article.parse()
# the replace is used to remove newlines
article_text = article.text.replace('\n', '')
print(article_text)
Less than a week ahead of the U.S. presidential election, misinformation relating to voting and 
election security is flourishing on Facebook, despite the platform's pledge to curb such content, 
a NewsGuard investigation has found. NewsGuard has identified 40 Facebook pages that are 
"super-spreaders" of election-related misinformation, meaning that they have shared false content 
about voting or the electoral process to their audiences of at least 100,000 followers. Only three 
of the 53 posts we identified on these pages—which together reach approximately 22.9 million 
followers—were flagged by Facebook as false. Four of the pages have managers based outside the 
U.S.—in Mexico,Vietnam, Australia, and Israel—despite the pages' focus on American politics. 
The myths identified by NewsGuard include false claims of mail-in ballots getting thrown away, 
narratives that dead people's cast ballots count as votes, and false claims about poll watchers. 
The claims about poll watchers cut both ways, with players on both the right and the left pushing 
their own, self-serving myths, NewsGuard found.NewsGuard's analysis also found that election-related 
myths often seize on routine and solvable voting errors as examples of malpractice or deception, 
sowing distrust in the electoral process. Others seem based on either an unintentional or willful 
misunderstanding of rules and practices.The false stories NewsGuard identified sometimes included 
multiple election myths, while other articles did not fit neatly with one particular election myth. 
Nevertheless, all the articles NewsGuard identified advanced inaccurate information about the voting 
process. For example, one popular Facebook post recently claimed that Pennsylvania had rejected 
372,000 ballots, when in fact, Pennsylvania officials had actually rejected 372,000 ballot applications. 
The rejection of absentee ballot applications is not uncommon, nor is it necessarily evidence of anything 
untoward. Moreover, a registered voter whose application to vote by mail was rejected can still vote in 
person. This falsehood appeared in an article published on 100Percent FedUp.com, a NewsGuard Red-rated 
(or generally unreliable) site. Patty McMurray, the co-owner of the site and the author of the article, 
told NewsGuard that her site had corrected the article to reflect the distinction between ballots and 
ballot applications. However, the false, uncorrected post remains accessible on Facebook and appears on 
at least five large Facebook pages. This claim was one of dozens that Facebook did not flag as false. 
When a Utah county accidentally sent out 13,000 absentee ballots without a signature line, the NewsGuard 
Red-rated site LawEnforcementToday.com called this a "cheat-by-mail scheme." The Salt Lake Tribune reported 
that the Sanpete County Clerk quickly learned of the mistake, which was a printing error, and immediately 
put information online explaining to voters how to correctly submit their ballot. There was no evidence 
that the mistake was part of a voter fraud scheme. But on October 15, the post was shared to three connected 
Facebook pages, with a total reach of 1.1 million followers. None of the posts were marked as false by 
Facebook's fact-checkers.Conspiratorial stories abounded, with articles warning of violence or other disastrous 
and unlawful election outcomes with no evidence to support their claims. Greg Palast, a liberal investigative 
journalist, predicted that 6 million people will vote by mail in Florida, but claimed their votes will likely 
not be counted. "The GOP-controlled Florida Legislature will say, we can't count them in time, so we're not 
going to certify the election," Palast wrote, suggesting this move would be part of a ploy to send the decision 
to the U.S. House, which under the 12th Amendment decides the president if no majority is reached in the electoral 
college.There is no evidence to suggest that the Florida legislature will refuse to certify the state's results. 
This article, shared on Facebook to Palast's 109,000 followers, was not flagged as false by Facebook. The three 
Facebook posts that were flagged by fact-checkers did not include such warnings until after the myth had been 
published and shared, due to the platform's practice of not providing advance warnings to users about pages that 
have been known to publish misinformation or hoaxes in the past. Had such warnings existed, Facebook users would 
have known in advance that they might be exposed to misinformation when reading those pages' posts.Despite Facebook's 
announced efforts to stop the spread of this type of misinformation, these pages continue to be allowed to publish 
blatant misinformation about voting and the electoral process — seemingly in violation of the platform's content 
policies. New false stories emerge daily, with inaccurate and deceptive interpretations of events that are perfectly 
normal. The result is that Facebook has exposed tens of millions of Americans to falsehoods about America's 
electoral process.

from newspaper import Config
from newspaper import Article
from utilities.nlp_utilities import NLPCustomMethods

nlp = NLPCustomMethods()

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'

config = Config()
config.browser_user_agent = USER_AGENT
config.request_timeout = 10

url = 'https://www.newsweek.com/facebook-super-spreader-election-misinformation-1543306'
article = Article(url, config=config)
article.download()
article.parse()
# the replace is used to remove newlines
article_text = article.text.replace('\n', '')

remove_stopwords = nlp.expunge_stopwords(article_text)
normalize_text = nlp.expunge_punctuations(remove_stopwords)

most_common_words = nlp.get_most_common_words(normalize_text, 20)
print(most_common_words)
[('facebook', 15), ('false', 9), ('newsguard', 8), ('pages', 8), ('election', 6), ('misinformation', 6), 
('voting', 5), ('identified', 5), ('electoral', 5), ('ballots', 5), ('shared', 4), ('process', 4), 
('myths', 4), ('claims', 4), ('ballot', 4), ('evidence', 4), ('article', 4), ('site', 4), ('platform', 3), 
('content', 3)]

# this output was sorted() and put into a set()
# noun types can also be tweak under NLPCustomMethods().get_nouns
nouns = nlp.get_nouns(normalize_text)
print(nouns)
['absentee', 'advance', 'amendment', 'americans', 'analysis', 'anything', 'application', 'applications', 
'article', 'articles', 'audiences', 'author', 'ballot', 'ballots', 'certify', 'cheatbymail', 'claims', 
'clerk', 'content', 'coowner', 'count', 'county', 'curb', 'deception', 'decides', 'decision', 'distinction', 
'dozens', 'efforts', 'election', 'error', 'errors', 'events', 'evidence', 'example', 'facebook', 'fact', 
'factcheckers', 'falsehood', 'falsehoods', 'falsewhen', 'flag', 'florida', 'followers', 'fraud', 'hoaxes', 
'house', 'information', 'interpretations', 'investigation', 'journalist', 'lawenforcementtodaycom', 'legislature', 
'line', 'mail', 'majority', 'malpractice', 'managers', 'mcmurray', 'meaning', 'mexicovietnam', 'millions', 
'misinformation', 'misunderstanding', 'move', 'myth', 'myths', 'narratives', 'officials', 'online', 'others', 
'pages', 'palast', 'part', 'pennsylvania', 'people', 'person', 'platform', 'players', 'pledge', 'policies', 
'politicsthe', 'poll', 'post', 'posts', 'practice', 'president', 'printing', 'process', 'processfor', 'reading', 
'reflect', 'refuse', 'result', 'results', 'rules', 'scheme', 'security', 'send', 'signature', 'site', 'spread', 
'state', 'stories', 'superspreaders', 'support', 'tens', 'time', 'tribune', 'users', 'violation', 'violence', 
'vote', 'voter', 'voters', 'votes', 'voting', 'warnings', 'watchers', 'ways', 'week']


# this output was sorted() and put into a set()
# verb types can also be tweak under NLPCustomMethods().get_verbs
verbs = nlp.get_verbs(normalize_text)
print(verbs)
['abounded', 'allowed', 'announced', 'appeared', 'appears', 'articles', 'australia', 'based', 'called', 
'cast', 'claim', 'claimed', 'connected', 'continue', 'corrected', 'counted', 'cut', 'electionrelated', 
'emerge', 'examples', 'existed', 'explaining', 'exposed', 'fit', 'flagged', 'flourishing', 'focus', 'found', 
'foundnewsguard', 'getting', 'going', 'greg', 'identified', 'include', 'known', 'learned', 'least', 'left', 
'mailin', 'marked', 'mistake', 'october', 'outcomes', 'ploy', 'poll', 'postsdespite', 'practicesthe', 'predicted', 
'providing', 'published', 'pushing', 'put', 'reach', 'reached', 'redrated', 'rejected', 'rejection', 'relating', 
'remains', 'reported', 'salt', 'say', 'seem', 'seize', 'selfserving', 'sent', 'shared', 'sowing', 'stop', 
'submit', 'suggesting', 'thrown', 'told', 'vote', 'voting', 'warning', 'wrote']

word_frequency = nlp.get_frequency_distribution(normalize_text, 20)
print(word_frequency)
[('facebook', 15), ('false', 9), ('newsguard', 8), ('pages', 8), ('election', 6), ('misinformation', 6), 
('voting', 5), ('identified', 5), ('electoral', 5), ('ballots', 5), ('shared', 4), ('process', 4), 
('myths', 4), ('claims', 4), ('ballot', 4), ('evidence', 4), ('article', 4), ('site', 4), ('platform', 3), 
('content', 3)]