From 116e858cc267cb1efaf3e9407f0c8ff37ea92d70 Mon Sep 17 00:00:00 2001 From: Sarah Date: Sat, 26 Mar 2022 00:33:39 +0000 Subject: [PATCH] add start of wiki update script --- update_wiki.py | 74 +++++++++++++++++++++++++++++++++++ vcinema_utils/VCinemaUtils.py | 15 ++++--- 2 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 update_wiki.py diff --git a/update_wiki.py b/update_wiki.py new file mode 100644 index 0000000..0d2d398 --- /dev/null +++ b/update_wiki.py @@ -0,0 +1,74 @@ +from bookstack import Bookstack +from update_films_by_country import get_films_by_country, build_page as build_films_by_country_page, FILMS_BY_COUNTRY_PAGE_ID +from update_films_by_year import get_films_by_year, build_page as build_films_by_year_page, FILMS_BY_YEAR_PAGE_ID +from vcinema_utils import VCinemaUtils +from update_viewings_csv import update_viewings_csv + +import argparse +from progress.bar import IncrementalBar + + +def update_wiki(token_id, token_secret, update_csv=False, update_films_by_year=True, update_films_by_country=True): + if update_csv: + print("Updating CSV") + update_viewings_csv(token_id, token_secret) + + print("Getting viewings") + viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret) + + data_fields = [] + if update_films_by_year: + data_fields.append("year") + + if update_films_by_country: + data_fields.append("countries") + + viewing_count = len(viewings) + with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: + VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar) + + update_page_count = 0 + if update_films_by_year: + update_page_count += 1 + if update_films_by_country: + update_page_count += 1 + + with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: + if update_films_by_year: + films_by_year = get_films_by_year(viewings) + bar.next() + if update_films_by_country: + films_by_country = get_films_by_country(viewings) + bar.next() + + bar.finish() + + with IncrementalBar('Generating pages', max=update_page_count, check_tty=False) as bar: + if update_films_by_year: + films_by_year_page = build_films_by_year_page(films_by_year) + bar.next() + if update_films_by_country: + films_by_country_page = build_films_by_country_page(films_by_country) + bar.next() + + bar.finish() + + with IncrementalBar('Updating pages', max=update_page_count, check_tty=False) as bar: + if update_films_by_year: + Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, FILMS_BY_YEAR_PAGE_ID, films_by_year_page) + bar.next() + if update_films_by_country: + Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, FILMS_BY_COUNTRY_PAGE_ID, films_by_country_page) + bar.next() + + bar.finish() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Update wiki pages.') + parser.add_argument('token_id', help='API token ID.') + parser.add_argument('token_secret', help='API token secret.') + + args = parser.parse_args() + + update_wiki(args.token_id, args.token_secret) diff --git a/vcinema_utils/VCinemaUtils.py b/vcinema_utils/VCinemaUtils.py index 88e860f..cbc5e10 100644 --- a/vcinema_utils/VCinemaUtils.py +++ b/vcinema_utils/VCinemaUtils.py @@ -9,15 +9,15 @@ from bookstack import Bookstack JACKNET_WIKI_URL = "https://wiki.jacknet.io" +# Page ID of https://wiki.jacknet.io/books/vcinema/page/csv +CSV_PAGE_ID = 11 def get_viewings_csv_attachment_id(token_id, token_secret): attachments = Bookstack.get_attachments(JACKNET_WIKI_URL, token_id, token_secret) - # Page ID of "https://wiki.jacknet.io/books/vcinema/page/csv" - page_id = 11 viewings_csv_file_name = "vcinema.csv" - return next((x['id'] for x in attachments if x['uploaded_to'] == page_id and x['name'] == viewings_csv_file_name), None) + return next((x['id'] for x in attachments if x['uploaded_to'] == CSV_PAGE_ID and x['name'] == viewings_csv_file_name), None) def get_vcinema_viewings(token_id, token_secret, viewings_csv=None, combine_repeat_viewings=True): @@ -63,13 +63,13 @@ def increment_progressbar(bar, _): def add_imdb_data_to_viewings(viewings, field_names, progressbar=None): with ThreadPoolExecutor(4) as executor: - future_to_url = {executor.submit(IMDbUtils.get_movie, viewing['imdb_id']) for viewing in viewings} + future_imdb_tasks = {executor.submit(IMDbUtils.get_movie, viewing['imdb_id']) for viewing in viewings} if progressbar is not None: - for this_future in future_to_url: + for this_future in future_imdb_tasks: this_future.add_done_callback(functools.partial(increment_progressbar, progressbar)) - for future in as_completed(future_to_url): + for future in as_completed(future_imdb_tasks): imdb_data = future.result() for viewing in viewings: @@ -78,6 +78,9 @@ def add_imdb_data_to_viewings(viewings, field_names, progressbar=None): if field_name in imdb_data: viewing[field_name] = imdb_data[field_name] + if progressbar is not None: + progressbar.finish() + def filter_viewings(viewings, filter_field, progressbar=None): viewings_filtered = {}