From 6c5162c6c3c6ef3be94b804393f114ec9bf44535 Mon Sep 17 00:00:00 2001 From: Sarah Date: Fri, 8 Apr 2022 23:27:09 +0100 Subject: [PATCH] add hidden_themes and keyword_scores in update_Wiki script --- update_keywords_scores.py | 2 +- update_wiki.py | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/update_keywords_scores.py b/update_keywords_scores.py index 7a78097..d9710a1 100644 --- a/update_keywords_scores.py +++ b/update_keywords_scores.py @@ -56,7 +56,7 @@ def add_keyword_scores(keyword_data, min_vcinema_count, min_imdb_count): keyword_data[keyword]['score'] = score -def build_table(keyword_data, minimum_score=1.0): +def build_page(keyword_data, minimum_score=1.0): keyword_data = {k: v for k, v in keyword_data.items() if 'score' in v and v['score'] >= minimum_score} keyword_data = OrderedDict(sorted(keyword_data.items(), key=lambda t: t[1]['score'], reverse=True)) diff --git a/update_wiki.py b/update_wiki.py index d1f0d0e..008ae7d 100644 --- a/update_wiki.py +++ b/update_wiki.py @@ -2,6 +2,8 @@ from bookstack import Bookstack from update_films_by_country import get_films_by_country, build_page as build_films_by_country_page, FILMS_BY_COUNTRY_PAGE_ID from update_film_references import get_films_by_reference, build_page as build_film_references_page, FILM_BY_REFERENCES_PAGE_ID from update_films_by_year import get_films_by_year, build_page as build_films_by_year_page, FILMS_BY_YEAR_PAGE_ID +from update_hidden_themes import get_hidden_themes, build_page as build_hidden_themes_page, HIDDEN_THEMES_PAGE_ID +from update_keywords_scores import get_keyword_scores, build_page as build_keyword_scores_page, KEYWORD_SCORES_PAGE_ID from vcinema_utils import VCinemaUtils from update_viewings_csv import update_viewings_csv @@ -9,7 +11,8 @@ import argparse from progress.bar import IncrementalBar -def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update_films_by_country, update_film_references): +def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update_films_by_country, + update_film_references, update_hidden_themes, update_keyword_scores): if update_csv: print("Updating CSV") update_viewings_csv(token_id, token_secret) @@ -24,14 +27,15 @@ def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update if update_films_by_country: data_fields.append("countries") - if update_film_references: + if update_film_references or update_hidden_themes or update_keyword_scores: data_fields.append("keywords") viewing_count = len(viewings) with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar) - update_page_count = sum([update_films_by_year, update_films_by_country, update_film_references]) + update_page_count = sum([update_films_by_year, update_films_by_country, update_film_references, + update_hidden_themes, update_keyword_scores]) with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar: if update_films_by_year: @@ -42,6 +46,14 @@ def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update bar.next() if update_film_references: films_by_reference = get_films_by_reference(viewings) + bar.next() + if update_hidden_themes: + hidden_themes = get_hidden_themes(viewings, token_id, token_secret) + bar.next() + if update_keyword_scores: + keyword_scores = get_keyword_scores(viewings) + bar.next() + bar.finish() @@ -54,6 +66,13 @@ def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update bar.next() if update_film_references: films_by_reference_page = build_film_references_page(films_by_reference) + bar.next() + if update_hidden_themes: + hidden_themes_page = build_hidden_themes_page(hidden_themes) + bar.next() + if update_keyword_scores: + keyword_scores_page = build_keyword_scores_page(keyword_scores) + bar.next() bar.finish() @@ -67,6 +86,12 @@ def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update if update_film_references: Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, FILM_BY_REFERENCES_PAGE_ID, markdown=films_by_reference_page) bar.next() + if update_hidden_themes: + Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, HIDDEN_THEMES_PAGE_ID, markdown=hidden_themes_page) + bar.next() + if update_keyword_scores: + Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, KEYWORD_SCORES_PAGE_ID, markdown=keyword_scores_page) + bar.next() bar.finish() @@ -80,7 +105,11 @@ if __name__ == '__main__': parser.add_argument('update_films_by_year', help='Update films by release year page, default: True', default=True) parser.add_argument('update films_by_country', help='Update films by country page, default: True', default=True) parser.add_argument('update_films_by_reference', help='Update films by reference page, default: True', default=True) + parser.add_argument('update_hidden_themes', help='Update hidden themese page, default: True', default=True) + parser.add_argument('update_keyword_scores', help='Upate keyword scores page - THIS TAKES A WHILE, default: False', default=False) args = parser.parse_args() - update_wiki(args.token_id, args.token_secret, args.update_csv, args.update_films_by_year, args.update_films_by_country, args.update_films_by_reference) + update_wiki(args.token_id, args.token_secret, args.update_csv, args.update_films_by_year, + args.update_films_by_country, args.update_films_by_reference, args.update_hidden_themes, + args.update_keyword_scores)