update_film_references.py

@@ -17,15 +17,11 @@ def get_films_by_reference(viewings):
 def build_page(films_by_reference_keyword):
     reference_keywords_sorted = OrderedDict(sorted(films_by_reference_keyword.items(), key=lambda t: t[0]))
 
-    table = "| Referenced | Films |\n| - | - |"
+    films_by_reference_table = "| Referenced | Films |\n| - | - |"
 
     for year in reference_keywords_sorted.keys():
-        table += "\n"
+        films_by_reference_table += "\n"
+        films_by_reference_table += str(string.capwords(year[13:].replace("-", " "))) + " | "
+        films_by_reference_table += VCinemaUtils.get_film_list(reference_keywords_sorted[year])
 
-        row_data = []
-        row_data.append(str(string.capwords(year[13:].replace("-", " "))))
-        row_data.append(VCinemaUtils.get_film_list(reference_keywords_sorted[year]))
-
-        table += " | ".join(row_data)
-
-    return table
+    return films_by_reference_table

update_films_by_country.py

@@ -54,17 +54,14 @@ def build_table(films_by_country):
     for country, films in films_by_country_sorted.items():
         table += "\n"
 
-        row_data = []
+        table += country
 
-        country_label = country
         if country in flags.keys():
-            country_label += " "
-            country_label += flags[country]
+            table += " "
+            table += flags[country]
 
-        row_data.append(country_label)
-        row_data.append(VCinemaUtils.get_film_list(films))
-
-        table += " | ".join(row_data)
+        table += " | "
+        table += VCinemaUtils.get_film_list(films)
 
     return table
 

update_films_by_year.py

@@ -19,11 +19,7 @@ def build_page(films_by_year):
 
     for year in films_by_year_sorted.keys():
         page += "\n"
-
-        row_data = []
-        row_data.append(str(year))
-        row_data.append(VCinemaUtils.get_film_list(films_by_year_sorted[year]))
-
-        page += " | ".join(row_data)
+        page += str(year) + " | "
+        page += VCinemaUtils.get_film_list(films_by_year_sorted[year])
 
     return page

update_hidden_themes.py

@@ -1,63 +0,0 @@
-from collections import OrderedDict
-
-from vcinema_utils import VCinemaUtils
-
-# Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-reference
-HIDDEN_THEMES_PAGE_ID = 63
-
-
-def get_hidden_themes(viewings, token_id, token_secret):
-    # Bit horrible to need to request this again, but it affects the order of the result table
-    viewings_ungrouped = VCinemaUtils.get_vcinema_viewings(token_id, token_secret, combine_repeat_viewings=False)
-
-    # Copy keywords from grouped viewings to ungrouped viewings
-    for viewing_ungrouped in viewings_ungrouped:
-        for viewing in viewings:
-            if viewing['imdb_id'] == viewing_ungrouped['imdb_id']:
-                if 'keywords' in viewing:
-                    viewing_ungrouped['keywords'] = viewing['keywords']
-                    break
-
-    viewings_filtered_watch_date = VCinemaUtils.filter_viewings(viewings_ungrouped, "date_watched")
-
-    for date, viewings in viewings_filtered_watch_date.items():
-        viewing_dict = {"viewings": viewings}
-
-        viewings_filtered_watch_date[date] = viewing_dict
-
-    # Add hidden themes
-    for date, data in viewings_filtered_watch_date.items():
-        hidden_themes = set()
-
-        if len(data['viewings']) > 1:
-            viewings_keywords = []
-
-            for viewing in data['viewings']:
-                if 'keywords' in viewing:
-                    viewings_keywords.append(set(viewing['keywords']))
-
-            if len(viewings_keywords) > 1:
-                hidden_themes = set.intersection(*viewings_keywords)
-
-        if hidden_themes != set():
-            viewings_filtered_watch_date[date]['hidden_themes'] = list(hidden_themes)
-
-    return viewings_filtered_watch_date
-
-
-def build_page(hidden_themes):
-    hidden_themes = OrderedDict(sorted(hidden_themes.items(), key=lambda t: t[0]))
-
-    table = "| Date | Films | Hidden Themes |\n| - | - | - |"
-
-    for date, data in hidden_themes.items():
-        table += "\n"
-
-        row_data = []
-        row_data.append(str(date))
-        row_data.append(VCinemaUtils.get_film_list(data['viewings']))
-        row_data.append((k for k in sorted(data['hidden_themes'])) if 'hidden_themes' in data else "N/A")
-
-        table += " | ".join(row_data)
-
-    return table

update_keywords_scores.py

@@ -1,75 +0,0 @@
-from collections import OrderedDict
-from progress.bar import IncrementalBar
-import math
-from concurrent.futures import ThreadPoolExecutor
-
-from imdb_utils import IMDbUtils
-from vcinema_utils import VCinemaUtils
-
-# Page ID of https://wiki.jacknet.io/books/vcinema/page/keyword-scores
-KEYWORD_SCORES_PAGE_ID = 23
-
-
-def get_keyword_scores(viewings):
-    viewings_filtered_keyword = VCinemaUtils.filter_viewings(viewings, "keywords")
-
-    for keyword, viewings in viewings_filtered_keyword.items():
-        viewings_filtered_keyword[keyword] = {"vcinema_films": viewings}
-
-    min_vcinema_count = 2
-    min_imdb_count = 4
-
-    add_keyword_totals(viewings_filtered_keyword, min_vcinema_count)
-    add_keyword_scores(viewings_filtered_keyword, min_vcinema_count, min_imdb_count)
-
-    return viewings_filtered_keyword
-
-
-def add_keyword_totals(keywords, min_vcinema_count):
-    keyword_count = len([keyword for keyword in keywords.keys() if len(keywords[keyword]['vcinema_films']) >= min_vcinema_count])
-
-    with IncrementalBar(message='%(percent).1f%% - %(eta)ds remaining', max=keyword_count, check_tty=False) as bar:
-        with ThreadPoolExecutor(6) as executor:
-            for keyword, data in keywords.items():
-                if len(data['vcinema_films']) >= min_vcinema_count:
-                    executor.submit(add_keyword_total, keyword, keywords, bar)
-
-
-def add_keyword_total(keyword, keywords, progress_bar=None):
-    keyword_total = IMDbUtils.get_keyword_count(keyword)
-
-    keywords[keyword]['total'] = keyword_total
-
-    if progress_bar is not None:
-        progress_bar.next()
-
-
-def add_keyword_scores(keyword_data, min_vcinema_count, min_imdb_count):
-    for keyword in keyword_data.keys():
-        if 'total' in keyword_data[keyword]:
-            vcinema_count = len(keyword_data[keyword]['vcinema_films'])
-            total_count = keyword_data[keyword]['total']
-
-            if vcinema_count >= min_vcinema_count and total_count >= min_imdb_count:
-                score = vcinema_count / math.log(total_count)
-
-                keyword_data[keyword]['score'] = score
-
-
-def build_table(keyword_data, minimum_score=1.0):
-    keyword_data = {k: v for k, v in keyword_data.items() if 'score' in v and v['score'] >= minimum_score}
-    keyword_data = OrderedDict(sorted(keyword_data.items(), key=lambda t: t[1]['score'], reverse=True))
-
-    table = "| Keyword | Number of VCinema Films | Total IMDb entries | Score |\n| - | - | - | - |"
-
-    for keyword, data in keyword_data.items():
-        table += "\n"
-
-        row_data = []
-        row_data.append(str(keyword))
-        row_data.append(str(len(data['vcinema_films'])))
-        row_data.append(str(len(data['total'])))
-        row_data.append(str(round(data['score'], 3)))
-        table += " | ".join(row_data)
-
-    return table

update_wiki.py

@@ -27,7 +27,11 @@ def update_wiki(token_id, token_secret, update_csv, update_films_by_year, update
     with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar)
 
-    update_page_count = sum([update_films_by_year, update_films_by_country])
+    update_page_count = 0
+    if update_films_by_year:
+        update_page_count += 1
+    if update_films_by_country:
+        update_page_count += 1
 
     with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         if update_films_by_year: