NewsGrabber/run.py

import logging
import sqlite3
import time
from pathlib import Path
from sources.ArsTechnica import ArsTechnica
from sources.BBCBusiness import BBCBusiness
from sources.BBCTechnology import BBCTechnology
from sources.Engadget import Engadget
from sources.FinancialTimes import FinancialTimes
from sources.ForbesBusiness import ForbesBusiness
from sources.ForbesFinance import ForbesFinance
from sources.ForbesTechnology import ForbesTechnology
from sources.HuffingtonPostBusiness import HuffingtonPostBusiness
from sources.HuffingtonPostTechnology import HuffingtonPostTechnology
from sources.MacWorld import MacWorld
from sources.NYTBusiness import NYTBusiness
from sources.NYTTechnology import NYTTechnology
from sources.ReutersBusiness import ReutersBusiness
from sources.ReutersTechnology import ReutersTechnology
from sources.TheEconomistBusiness import TheEconomistBusiness
from sources.TheEconomistFinance import TheEconomistFinance
from sources.TheEconomistTechnology import TheEconomistTechnology
from sources.Wired import Wired

DATABASE_PATH = Path("storage.db")
GRAB_FREQUENCY = 60


def main():
    try:
        db = sqlite3.connect(DATABASE_PATH)
        if not db:
            raise sqlite3.DatabaseError

        grabbers = [
            ArsTechnica(db),
            BBCBusiness(db),
            BBCTechnology(db),
            Engadget(db),
            FinancialTimes(db),
            ForbesBusiness(db),
            ForbesFinance(db),
            ForbesTechnology(db),
            HuffingtonPostBusiness(db),
            HuffingtonPostTechnology(db),
            MacWorld(db),
            NYTBusiness(db),
            NYTTechnology(db),
            ReutersBusiness(db),
            ReutersTechnology(db),
            TheEconomistBusiness(db),
            TheEconomistFinance(db),
            TheEconomistTechnology(db),
            Wired(db),
        ]

        while True:
            for grabber in grabbers:
                grabber.grab()
                time.sleep(GRAB_FREQUENCY/len(grabbers))

    except sqlite3.Error:
        logging.error("Could not connect to database.")
        exit(-1)


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO,format="%(asctime)s %(levelname)-8s %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
    main()