NewsGrabber/run.py

import logging
import sqlite3
import time
from pathlib import Path
from sources.ArsTechnica import ArsTechnica
from sources.BBCBusiness import BBCBusiness
from sources.BBCTechnology import BBCTechnology
from sources.ReutersBusiness import ReutersBusiness
from sources.ReutersTechnology import ReutersTechnology
from sources.NYTBusiness import NYTBusiness
from sources.NYTTechnology import NYTTechnology
from sources.ForbesBusiness import ForbesBusiness
from sources.ForbesTechnology import ForbesTechnology
from sources.FinancialTimes import FinancialTimes
from sources.MacWorld import MacWorld
from sources.Wired import Wired
from sources.Engadget import Engadget
from sources.HuffingtonPostBusiness import HuffingtonPostBusiness
from sources.HuffingtonPostTechnology import HuffingtonPostTechnology

DATABASE_PATH = Path("storage.db")
GRAB_FREQUENCY = 15
GRAB_INTERVAL = 5


def main():
    try:
        db = sqlite3.connect(DATABASE_PATH)
        if not db:
            raise sqlite3.DatabaseError

        grabbers = [
            ArsTechnica(db),
            BBCBusiness(db),
            BBCTechnology(db),
            ReutersBusiness(db),
            ReutersTechnology(db),
            NYTBusiness(db),
            NYTTechnology(db),
            ForbesBusiness(db),
            ForbesTechnology(db),
            FinancialTimes(db),
            MacWorld(db),
            Wired(db),
            Engadget(db),
            HuffingtonPostBusiness(db),
            HuffingtonPostTechnology(db),
        ]

        while True:
            for grabber in grabbers:
                grabber.grab()
                time.sleep(GRAB_FREQUENCY/GRAB_INTERVAL)

    except sqlite3.Error:
        logging.error("Could not connect to database.")
        exit(-1)


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO,format="%(asctime)s %(levelname)-8s %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
    main()