37 lines
718 B
Python
37 lines
718 B
Python
import logging
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from sources.ArsTechnica import ArsTechnica
|
|
from sources.BBCBusiness import BBCBusiness
|
|
|
|
DATABASE_PATH = Path("storage.db")
|
|
SCRAPE_INTERVAL = 15
|
|
|
|
|
|
def configure_logging():
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)-8s %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M"
|
|
)
|
|
|
|
|
|
def setup_database():
|
|
db = sqlite3.connect(DATABASE_PATH)
|
|
return db
|
|
|
|
|
|
def main():
|
|
db = setup_database()
|
|
grabbers = [
|
|
ArsTechnica(db, SCRAPE_INTERVAL),
|
|
BBCBusiness(db, SCRAPE_INTERVAL)
|
|
]
|
|
for grabber in grabbers:
|
|
grabber.start()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
configure_logging()
|
|
main()
|