From 0a5247691fff15f9ca9256ebe673d345508c7f4b Mon Sep 17 00:00:00 2001 From: Sarah Date: Mon, 27 Jun 2022 00:11:06 +0100 Subject: [PATCH] read token from file --- test/test_VCinemaUtils.py | 0 update_wiki.py | 67 ++++++++++++++++++++++++++------------ wiki_pages/ActorScores.py | 26 +++++++++++++++ wiki_pages/HiddenThemes.py | 19 ++++++----- 4 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 test/test_VCinemaUtils.py create mode 100644 wiki_pages/ActorScores.py diff --git a/test/test_VCinemaUtils.py b/test/test_VCinemaUtils.py new file mode 100644 index 0000000..e69de29 diff --git a/update_wiki.py b/update_wiki.py index 0b0e936..5c252f1 100644 --- a/update_wiki.py +++ b/update_wiki.py @@ -2,6 +2,7 @@ from wiki_pages import FilmsByCountry, FilmsByReference, FilmsByYear, HiddenThem from vcinema_utils import VCinemaUtils import argparse +import json from progress.bar import IncrementalBar @@ -33,35 +34,59 @@ def update_wiki(token_id, token_secret, update_csv, pages): with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar) - print("Processing viewing data") + update_page_count = sum([update_films_by_year, update_films_by_country, update_film_references, + update_hidden_themes, update_keyword_scores]) - if update_films_by_year: - films_by_year = FilmsByYear.get_films_by_year(viewings) - FilmsByYear.update_page(token_id, token_secret, films_by_year) - if update_films_by_country: - films_by_country = FilmsByCountry.get_films_by_country(viewings) - FilmsByCountry.update_page(token_id, token_secret, films_by_country) - if update_film_references: - films_by_reference = FilmsByReference.get_films_by_reference(viewings) - FilmsByReference.update_page(token_id, token_secret, films_by_reference) - if update_hidden_themes: - hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret) - HiddenThemes.update_page(token_id, token_secret, hidden_themes) - if update_keyword_scores: - keyword_scores = KeywordScores.get_keyword_scores(viewings) - KeywordScores.update_page(token_id, token_secret, keyword_scores) + with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: + if update_films_by_year: + films_by_year = FilmsByYear.get_films_by_year(viewings) + bar.next() + if update_films_by_country: + films_by_country = FilmsByCountry.get_films_by_country(viewings) + bar.next() + if update_film_references: + films_by_reference = FilmsByReference.get_films_by_reference(viewings) + bar.next() + if update_hidden_themes: + hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret) + bar.next() + if update_keyword_scores: + keyword_scores = KeywordScores.get_keyword_scores(viewings) + bar.next() - print("Done!") + bar.finish() + + with IncrementalBar('Updating pages', max=update_page_count, check_tty=False) as bar: + if update_films_by_year: + FilmsByYear.update_page(token_id, token_secret, films_by_year) + bar.next() + if update_films_by_country: + FilmsByCountry.update_page(token_id, token_secret, films_by_country) + bar.next() + if update_film_references: + FilmsByReference.update_page(token_id, token_secret, films_by_reference) + bar.next() + if update_hidden_themes: + HiddenThemes.update_page(token_id, token_secret, hidden_themes) + bar.next() + if update_keyword_scores: + KeywordScores.update_page(token_id, token_secret, keyword_scores) + bar.next() + + bar.finish() if __name__ == '__main__': parser = argparse.ArgumentParser(description='Update wiki pages.') - parser.add_argument('token_id', help='API token ID.') - parser.add_argument('token_secret', help='API token secret.') - parser.add_argument("--do_not_update_csv", action="store_true") + parser.add_argument('--update_csv', help='Update viewings.csv file, default: True', default=True, required=False) parser.add_argument('--pages', nargs="+", default=['years', 'countries', 'references', 'themes', 'scores'], required=False) args = parser.parse_args() - update_wiki(args.token_id, args.token_secret, not args.do_not_update_csv, args.pages) + with open('token.json') as json_file: + token = json.load(json_file) + + update_wiki(token['token_id'], token['token_secret'], not args.do_not_update_csv, args.pages) + + update_wiki(args.token_id, args.token_secret, args.update_csv, args.pages) diff --git a/wiki_pages/ActorScores.py b/wiki_pages/ActorScores.py new file mode 100644 index 0000000..4869867 --- /dev/null +++ b/wiki_pages/ActorScores.py @@ -0,0 +1,26 @@ +from collections import Counter, OrderedDict +import warnings + +from vcinema_utils import VCinemaUtils + +warnings.filterwarnings("ignore") + + +def get_actor_scores(viewings): + for viewing_ind in range(len(viewings)): + viewing = viewings[viewing_ind] + + cast_list = [] + + for cast_member in viewing['cast']: + cast_member_name = (cast_member['name'], cast_member.currentRole) + cast_list.append(cast_member_name) + + viewings[viewing_ind]['cast'] = cast_list + + viewings_filtered_by_cast_member = VCinemaUtils.filter_viewings(viewings, "cast") + + viewings_filtered_by_cast_member = OrderedDict(sorted(viewings_filtered_by_cast_member.items(), key=lambda t: len(t[1]), reverse=True)) + + return viewings_filtered_by_cast_member + diff --git a/wiki_pages/HiddenThemes.py b/wiki_pages/HiddenThemes.py index 6a8fb0c..06bf3c9 100644 --- a/wiki_pages/HiddenThemes.py +++ b/wiki_pages/HiddenThemes.py @@ -28,27 +28,28 @@ def get_hidden_themes(viewings, token_id, token_secret): # Add hidden themes for date, data in viewings_filtered_watch_date.items(): - hidden_themes = set() + keyword_counts = {} if len(data['viewings']) > 1: - viewings_keywords = [] - for viewing in data['viewings']: if 'keywords' in viewing: - viewings_keywords.append(set(viewing['keywords'])) + for keyword in viewing['keywords']: + if keyword in keyword_counts.keys(): + keyword_counts[keyword] += 1 + else: + keyword_counts[keyword] = 1 - if len(viewings_keywords) > 1: - hidden_themes = set.intersection(*viewings_keywords) + keyword_counts = {k: v for k, v in sorted(keyword_counts.items(), key=lambda item: item[1], reverse=True)} - if hidden_themes != set(): - viewings_filtered_watch_date[date]['hidden_themes'] = list(hidden_themes) + viewings_filtered_watch_date[date]['keyword_counts'] = keyword_counts return viewings_filtered_watch_date def update_page(token_id, token_secret, hidden_themes): page = build_page(hidden_themes) - Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page) + print(page) + # Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page) def build_page(hidden_themes):