70 lines
2.2 KiB
Python
70 lines
2.2 KiB
Python
import logging
|
|
import sqlite3
|
|
import time
|
|
from pathlib import Path
|
|
from sources.ArsTechnica import ArsTechnica
|
|
from sources.BBCBusiness import BBCBusiness
|
|
from sources.BBCTechnology import BBCTechnology
|
|
from sources.Engadget import Engadget
|
|
from sources.FinancialTimes import FinancialTimes
|
|
from sources.ForbesBusiness import ForbesBusiness
|
|
from sources.ForbesFinance import ForbesFinance
|
|
from sources.ForbesTechnology import ForbesTechnology
|
|
from sources.HuffingtonPostBusiness import HuffingtonPostBusiness
|
|
from sources.HuffingtonPostTechnology import HuffingtonPostTechnology
|
|
from sources.MacWorld import MacWorld
|
|
from sources.NYTBusiness import NYTBusiness
|
|
from sources.NYTTechnology import NYTTechnology
|
|
from sources.ReutersBusiness import ReutersBusiness
|
|
from sources.ReutersTechnology import ReutersTechnology
|
|
from sources.TheEconomistBusiness import TheEconomistBusiness
|
|
from sources.TheEconomistFinance import TheEconomistFinance
|
|
from sources.TheEconomistTechnology import TheEconomistTechnology
|
|
from sources.Wired import Wired
|
|
|
|
DATABASE_PATH = Path("storage.db")
|
|
GRAB_FREQUENCY = 60
|
|
|
|
|
|
def main():
|
|
try:
|
|
db = sqlite3.connect(DATABASE_PATH)
|
|
if not db:
|
|
raise sqlite3.DatabaseError
|
|
|
|
grabbers = [
|
|
ArsTechnica(db),
|
|
BBCBusiness(db),
|
|
BBCTechnology(db),
|
|
Engadget(db),
|
|
FinancialTimes(db),
|
|
ForbesBusiness(db),
|
|
ForbesFinance(db),
|
|
ForbesTechnology(db),
|
|
HuffingtonPostBusiness(db),
|
|
HuffingtonPostTechnology(db),
|
|
MacWorld(db),
|
|
NYTBusiness(db),
|
|
NYTTechnology(db),
|
|
ReutersBusiness(db),
|
|
ReutersTechnology(db),
|
|
TheEconomistBusiness(db),
|
|
TheEconomistFinance(db),
|
|
TheEconomistTechnology(db),
|
|
Wired(db),
|
|
]
|
|
|
|
while True:
|
|
for grabber in grabbers:
|
|
grabber.grab()
|
|
time.sleep(GRAB_FREQUENCY/len(grabbers))
|
|
|
|
except sqlite3.Error:
|
|
logging.error("Could not connect to database.")
|
|
exit(-1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO,format="%(asctime)s %(levelname)-8s %(message)s",datefmt="%Y-%m-%d %H:%M:%S")
|
|
main()
|