Merge pull request 'fix bug where script throws exception if no page exists for a reference keyword' (#14) from bugfix/exception-thrown-if-no-page-exists-for-reference into master

Reviewed-on: #14

.gitignore

@@ -0,0 +1,5 @@
+
+*.pyc
+.idea/*
+__pycache__/*
+.DS_Store

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "wiki_utils"]
-	path = wiki_utils
-	url = gitea@git.jacknet.io:sarah/wiki_utils.git
+[submodule "bookstack"]
+	path = bookstack
+	url = git@git.jacknet.io:sarah/bookstack.git

bookstack

@@ -0,0 +1 @@
+Subproject commit 8f6e38cb337bcf51b0790f2db7001681ceb9338f

country-flags.csv

@@ -0,0 +1,258 @@
+Country,Flag
+Afghanistan,🇦🇫
+Åland Islands,🇦🇽
+Albania,🇦🇱
+Algeria,🇩🇿
+American Samoa,🇦🇸
+Andorra,🇦🇩
+Angola,🇦🇴
+Anguilla,🇦🇮
+Antarctica,🇦🇶
+Antigua and Barbuda,🇦🇬
+Argentina,🇦🇷
+Armenia,🇦🇲
+Aruba,🇦🇼
+Australia,🇦🇺
+Austria,🇦🇹
+Azerbaijan,🇦🇿
+Bahamas,🇧🇸
+Bahrain,🇧🇭
+Bangladesh,🇧🇩
+Barbados,🇧🇧
+Belarus,🇧🇾
+Belgium,🇧🇪
+Belize,🇧🇿
+Benin,🇧🇯
+Bermuda,🇧🇲
+Bhutan,🇧🇹
+Bolivia,🇧🇴
+"Bonaire, Sint Eustatius and Saba",🇧🇶
+Bosnia and Herzegovina,🇧🇦
+Botswana,🇧🇼
+Bouvet Island,🇧🇻
+Brazil,🇧🇷
+British Indian Ocean Territory,🇮🇴
+British Virgin Islands,🇻🇬
+Brunei Darussalamm,🇧🇳
+Bulgaria,🇧🇬
+Burkina Faso,🇧🇫
+Burma,🇲🇲
+Burundi,🇧🇮
+Cambodia,🇰🇭
+Cameroon,🇨🇲
+Canada,🇨🇦
+Cape Verde,🇨🇻
+Cayman Islands,🇰🇾
+Central African Republic,🇨🇫
+Chad,🇹🇩
+Chile,🇨🇱
+China,🇨🇳
+Christmas Island,🇨🇽
+Cocos (Keeling) Islands,🇨🇨
+Colombia,🇨🇴
+Comoros,🇰🇲
+Congo,🇨🇩
+Cook Islands,🇨🇰
+Costa Rica,🇨🇷
+Côte d'Ivoire,🇨🇮
+Croatia,🇭🇷
+Cuba,🇨🇺
+Cyprus,🇨🇾
+Czech Republic,🇨🇿
+Czechoslovakia,🇨🇿
+Democratic Republic of the Congo,🇨🇩
+Denmark,🇩🇰
+Djibouti,🇩🇯
+Dominica,🇩🇲
+Dominican Republic,🇩🇴
+East Germany,➡️🇩🇪
+Ecuador,🇪🇨
+Egypt,🇪🇬
+El Salvador,🇸🇻
+Equatorial Guinea,🇬🇶
+Eritrea,🇪🇷
+Estonia,🇪🇪
+Ethiopia,🇪🇹
+Falkland Islands,🇫🇰
+Faroe Islands,🇫🇴
+Federated States of Micronesia,🇫🇲
+Fiji,🇫🇯
+Finland,🇫🇮
+France,🇫🇷
+French Guiana,🇬🇫
+French Polynesia,🇵🇫
+French Southern Territories,🇹🇫
+Gabon,🇬🇦
+Gambia,🇬🇲
+Georgia,🇬🇪
+Germany,🇩🇪
+Ghana,🇬🇭
+Gibraltar,🇬🇮
+Greece,🇬🇷
+Greenland,🇬🇱
+Grenada,🇬🇩
+Guadeloupe,🇬🇵
+Guam,🇬🇺
+Guatemala,🇬🇹
+Guernsey,🇬🇬
+Guinea,🇬🇳
+Guinea-Bissau,🇬🇼
+Guyana,🇬🇾
+Haiti,🇭🇹
+Heard Island and McDonald Islands,🇭🇲
+Holy See (Vatican City State),🇻🇦
+Honduras,🇭🇳
+Hong Kong,🇭🇰
+Hungary,🇭🇺
+Iceland,🇮🇸
+India,🇮🇳
+Indonesia,🇮🇩
+Iran,🇮🇷
+Iraq,🇮🇶
+Ireland,🇮🇪
+Isle of Man,🇮🇲
+Israel,🇮🇱
+Italy,🇮🇹
+Jamaica,🇯🇲
+Japan,🇯🇵
+Jersey,🇯🇪
+Jordan,🇯🇴
+Kazakhstan,🇰🇿
+Kenya,🇰🇪
+Kiribati,🇰🇮
+Korea,🇰🇵🇰🇷
+Kosovo,🇽🇰
+Kuwait,🇰🇼
+Kyrgyzstan,🇰🇬
+Laos,🇱🇦
+Latvia,🇱🇻
+Lebanon,🇱🇧
+Lesotho,🇱🇸
+Liberia,🇱🇷
+Libya,🇱🇾
+Liechtenstein,🇱🇮
+Lithuania,🇱🇹
+Luxembourg,🇱🇺
+Macao,🇲🇴
+Madagascar,🇲🇬
+Malawi,🇲🇼
+Malaysia,🇲🇾
+Maldives,🇲🇻
+Mali,🇲🇱
+Malta,🇲🇹
+Marshall Islands,🇲🇭
+Martinique,🇲🇶
+Mauritania,🇲🇷
+Mauritius,🇲🇺
+Mayotte,🇾🇹
+Mexico,🇲🇽
+Moldova,🇲🇩
+Monaco,🇲🇨
+Mongolia,🇲🇳
+Montenegro,🇲🇪
+Montserrat,🇲🇸
+Morocco,🇲🇦
+Mozambique,🇲🇿
+Myanmar,🇲🇲
+Namibia,🇳🇦
+Nauru,🇳🇷
+Nepal,🇳🇵
+Netherlands,🇳🇱
+Netherlands Antilles,🇳🇱
+New Caledonia,🇳🇨
+New Zealand,🇳🇿
+Nicaragua,🇳🇮
+Niger,🇳🇪
+Nigeria,🇳🇬
+Niue,🇳🇺
+Norfolk Island,🇳🇫
+North Korea,🇰🇵
+North Vietnam,🇻🇳
+Northern Mariana Islands,🇲🇵
+Norway,🇳🇴
+Oman,🇴🇲
+Pakistan,🇵🇰
+Palau,🇵🇼
+Palestine,🇵🇸
+Palestinian Territory,🇵🇸
+Panama,🇵🇦
+Papua New Guinea,🇵🇬
+Paraguay,🇵🇾
+Peru,🇵🇪
+Philippines,🇵🇭
+Poland,🇵🇱
+Portugal,🇵🇹
+Pitcairn,🇵🇳
+Puerto Rico,🇵🇷
+Qatar,🇶🇦
+Republic of Macedonia,🇲🇰
+Réunion,🇷🇪
+Romania,🇷🇴
+Russia,🇷🇺
+Rwanda,🇷🇼
+Saint Barthélemy,🇧🇱
+Saint Helena,🇸🇭
+Saint Kitts and Nevis,🇰🇳
+Saint Lucia,🇱🇨
+Saint Martin (French part),🇫🇷
+Saint Pierre and Miquelon,🇵🇲
+Saint Vincent and the Grenadines,🇻🇨
+Samoa,🇼🇸
+San Marino,🇸🇲
+Sao Tome and Principe,🇸🇹
+Saudi Arabia,🇸🇦
+Senegal,🇸🇳
+Serbia,🇷🇸
+Serbia and Montenegro,🇷🇸🇲🇪
+Seychelles,🇸🇨
+Siam,🇹🇭
+Sierra Leone,🇸🇱
+Singapore,🇸🇬
+Slovakia,🇸🇰
+Slovenia,🇸🇮
+Solomon Islands,🇸🇧
+Somalia,🇸🇴
+South Africa,🇿🇦
+South Georgia and the South Sandwich Islands,🇬🇸
+South Korea,🇰🇷
+Spain,🇪🇸
+Sri Lanka,🇱🇰
+Sudan,🇸🇩
+Suriname,🇸🇷
+Svalbard and Jan Mayen,🇸🇯
+Swaziland,🇸🇿
+Sweden,🇸🇪
+Switzerland,🇨🇭
+Syria,🇸🇾
+Taiwan,🇹🇼
+Tajikistan,🇹🇯
+Tanzania,🇹🇿
+Thailand,🇹🇭
+Timor-Leste,🇹🇱
+Togo,🇹🇬
+Tokelau,🇹🇰
+Tonga,🇹🇴
+Trinidad and Tobago,🇹🇹
+Tunisia,🇹🇳
+Turkey,🇹🇷
+Turkmenistan,🇹🇲
+Turks and Caicos Islands,🇹🇨
+Tuvalu,🇹🇻
+U.S. Virgin Islands,🇻🇮
+Uganda,🇺🇬
+Ukraine,🇺🇦
+United Arab Emirates,🇦🇪
+United Kingdom,🇬🇧
+United States,🇺🇸
+United States Minor Outlying Islands,🇺🇲
+Uruguay,🇺🇾
+Uzbekistan,🇺🇿
+Vanuatu,🇻🇺
+Venezuela,🇻🇪
+Vietnam,🇻🇳
+Wallis and Futuna,🇫🇷
+West Germany,⬅️🇩🇪
+Western Sahara,🇪🇭
+Yemen,🇾🇪
+Zambia,🇿🇲
+Zimbabwe,🇿🇼

generate_map_timelapse.py

@@ -0,0 +1,73 @@
+from wiki_pages import FilmsByCountry
+from vcinema_utils import VCinemaUtils
+
+import argparse
+from collections import OrderedDict
+import imageio
+from progress.bar import IncrementalBar
+from pygifsicle import optimize
+from PIL import Image, ImageFont, ImageDraw, ImageFont
+import io
+
+
+def generate_map_timelapse(token_id, token_secret, filename):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret, combine_repeat_viewings=False)
+
+    viewing_count = len(viewings)
+    with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ['countries'], bar)
+
+    date_viewings = VCinemaUtils.filter_viewings(viewings, "date_watched")
+
+    date_viewings = OrderedDict(sorted(date_viewings.items(), key=lambda t: t[0]))
+
+    running_country_counts = {}
+    print(len(date_viewings.keys()))
+
+    with imageio.get_writer(filename, mode='I', duration=0.1) as writer:
+        for date, viewings in date_viewings.items():
+            date_viewings_countries = VCinemaUtils.filter_viewings(viewings, "countries")
+
+            for country in date_viewings_countries:
+                if country in running_country_counts.keys():
+                    running_country_counts[country] += date_viewings_countries[country]
+                else:
+                    running_country_counts[country] = date_viewings_countries[country]
+
+            map = FilmsByCountry.draw_map(running_country_counts, file_name="map-{}.svg".format(date))
+
+            stream = io.BytesIO(map)
+            img = Image.open(stream)
+
+            map_editable = ImageDraw.Draw(img)
+
+            # macos font path
+            font = ImageFont.truetype("/System/Library/Fonts/Supplemental/Arial.ttf", 48)
+
+            # image is 655 high
+            map_editable.text((2, 605), "{}".format(date), (255, 64, 0), font=font)
+
+            img_byte_arr = io.BytesIO()
+            img.save(img_byte_arr, format='PNG')
+            img_byte_arr = img_byte_arr.getvalue()
+
+            image = imageio.imread(img_byte_arr)
+            writer.append_data(image)
+
+    print("optimizing")
+
+    optimize(filename)
+
+    print("done")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create timelapse gif of vcinema countries')
+    parser.add_argument('token_id', help='API token ID.')
+    parser.add_argument('token_secret', help='API token secret.')
+    parser.add_argument('filename', help='Name of output gif')
+
+    args = parser.parse_args()
+
+    generate_map_timelapse(args.token_id, args.token_secret, args.filename)

get_hidden_themes.py

@@ -0,0 +1,33 @@
+from imdb_utils import IMDbUtils
+
+import argparse
+from progress.bar import IncrementalBar
+
+
+def get_hidden_themes(imdb_ids):
+    film_keywords = []
+
+    with IncrementalBar('Retrieving movie data', max=len(imdb_ids), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        for imdb_id in imdb_ids:
+            movie_data = IMDbUtils.get_movie_keywords(imdb_id)
+
+            if 'keywords' in movie_data:
+                keywords = set(movie_data['keywords'])
+                film_keywords.append(keywords)
+
+            bar.next()
+
+    hidden_themes = set.intersection(*film_keywords)
+
+    return hidden_themes
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('imdb_ids', nargs="+", default=[])
+
+    args = parser.parse_args()
+
+    hidden_themes = get_hidden_themes(args.imdb_ids)
+
+    print(hidden_themes)

imdb_utils/IMDbUtils.py

@@ -0,0 +1,60 @@
+from imdb import IMDb
+import requests
+from bs4 import BeautifulSoup
+import re
+
+
+def get_movie(imdb_id):
+    ia = IMDb()
+
+    movie = ia.get_movie(imdb_id)
+
+    return movie
+
+
+def get_movie_keywords(imdb_id):
+    ia = IMDb()
+
+    movie = ia.get_movie(imdb_id, info="keywords")
+
+    return movie
+
+
+def get_api_keyword_count(keyword):
+    ia = IMDb()
+
+    count = len(ia.get_keyword(keyword))
+
+    return count
+
+
+def get_website_keyword_count(keyword):
+    try:
+        page = requests.get("https://www.imdb.com/search/keyword/?keywords=" + keyword)
+    except ConnectionError:
+        raise
+
+    soup = BeautifulSoup(page.content, 'html.parser')
+    elements = soup.findAll("div", class_="desc")
+
+    pagination_label = elements[0].text.replace("\n", "")
+
+    pagination_label_reg = "(\d+,?\d*) titles"
+    pattern_match = re.compile(pagination_label_reg).search(pagination_label)
+
+    if pattern_match is not None:
+        return int(pattern_match.group(1).replace(',', ''))
+    else:
+        return 1
+
+
+def get_keyword_count(keyword):
+    count = get_api_keyword_count(keyword)
+
+    if count == 50:
+        try:
+            count = get_website_keyword_count(keyword)
+        except Exception as e:
+            raise
+
+    return count

update_viewings_csv.py

@@ -0,0 +1,16 @@
+import json
+
+from wiki_pages import ViewingsCsv
+
+
+def update_viewings_csv(token_id, token_secret):
+    print("Updating CSV")
+    ViewingsCsv.update_viewings_csv(token_id, token_secret)
+    print("Done!")
+
+
+if __name__ == '__main__':
+    with open('token.json') as json_file:
+        token = json.load(json_file)
+
+    update_viewings_csv(token['token_id'], token['token_secret'])

update_wiki.py

@@ -0,0 +1,69 @@
+from wiki_pages import FilmsByCountry, FilmsByReference, FilmsByYear, HiddenThemes, KeywordScores, ViewingsCsv
+from vcinema_utils import VCinemaUtils
+
+import argparse
+import json
+from progress.bar import IncrementalBar
+
+
+def update_wiki(token_id, token_secret, update_csv, pages):
+    if update_csv:
+        print("Updating CSV")
+        ViewingsCsv.update_viewings_csv(token_id, token_secret)
+
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    update_films_by_year = 'years' in pages
+    update_films_by_country = 'countries' in pages
+    update_film_references = 'references' in pages
+    update_hidden_themes = 'themes' in pages
+    update_keyword_scores = 'scores' in pages
+
+    data_fields = []
+    if update_films_by_year:
+        data_fields.append("year")
+
+    if update_films_by_country:
+        data_fields.append("countries")
+
+    if update_film_references or update_hidden_themes or update_keyword_scores:
+        data_fields.append("keywords")
+
+    viewing_count = len(viewings)
+    with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar)
+
+    print("Processing viewing data")
+
+    if update_films_by_year:
+        films_by_year = FilmsByYear.get_films_by_year(viewings)
+        FilmsByYear.update_page(token_id, token_secret, films_by_year)
+    if update_films_by_country:
+        films_by_country = FilmsByCountry.get_films_by_country(viewings)
+        FilmsByCountry.update_page(token_id, token_secret, films_by_country)
+    if update_film_references:
+        films_by_reference = FilmsByReference.get_films_by_reference(viewings)
+        FilmsByReference.update_page(token_id, token_secret, films_by_reference)
+    if update_hidden_themes:
+        hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret)
+        HiddenThemes.update_page(token_id, token_secret, hidden_themes)
+    if update_keyword_scores:
+        keyword_scores = KeywordScores.get_keyword_scores(viewings)
+        KeywordScores.update_page(token_id, token_secret, keyword_scores)
+
+    print("Done!")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update wiki pages.')
+
+    parser.add_argument('--pages', nargs="+", default=['years', 'countries', 'references', 'themes', 'scores'], required=False)
+    parser.add_argument("--do_not_update_csv", action="store_true")
+
+    args = parser.parse_args()
+
+    with open('token.json') as json_file:
+        token = json.load(json_file)
+
+    update_wiki(token['token_id'], token['token_secret'], not args.do_not_update_csv, args.pages)

vcinema_utils/VCinemaUtils.py

@@ -0,0 +1,150 @@
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
+import csv
+
+from imdb_utils import IMDbUtils
+from bookstack import Bookstack
+
+
+JACKNET_WIKI_URL = "https://wiki.jacknet.io"
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/csv
+CSV_PAGE_ID = 11
+
+
+def get_viewings_csv_attachment_id(token_id, token_secret):
+    attachments = Bookstack.get_attachments(JACKNET_WIKI_URL, token_id, token_secret)
+
+    viewings_csv_file_name = "vcinema.csv"
+
+    return next((x['id'] for x in attachments if x['uploaded_to'] == CSV_PAGE_ID and x['name'] == viewings_csv_file_name), None)
+
+
+def get_vcinema_viewings(token_id, token_secret, viewings_csv=None, combine_repeat_viewings=True):
+    if viewings_csv is None:
+        attachment_id = get_viewings_csv_attachment_id(token_id, token_secret)
+        viewings_csv = Bookstack.get_attachment(JACKNET_WIKI_URL, token_id, token_secret, attachment_id)
+
+    viewings_csv = viewings_csv.decode("utf-8")
+    viewings_csv_rows = viewings_csv.strip().split("\n")
+
+    viewings = list(csv.DictReader(viewings_csv_rows, quotechar='"'))
+
+    if combine_repeat_viewings:
+        for viewing in viewings:
+            viewing['viewings'] = [
+                {'date_watched': viewing['date_watched'], 'season': viewing['season'], 'rating': viewing['rating']}]
+            viewing.pop('date_watched')
+            viewing.pop('season')
+            viewing.pop('rating')
+
+        watch_counts = Counter([x['imdb_id'] for x in viewings])
+        repeat_watches = [k for k, v in watch_counts.items() if v > 1]
+
+        for film in repeat_watches:
+            viewing_indexes = [index for index, viewing in enumerate(viewings) if viewing['imdb_id'] == film]
+
+            first_watch = viewings[viewing_indexes[0]]
+
+            for index in viewing_indexes[1::]:
+                first_watch['viewings'].extend(viewings[index]['viewings'])
+
+            for index in reversed(viewing_indexes[1::]):
+                viewings.pop(index)
+
+    return viewings
+
+
+def add_imdb_data(imdb_id, viewings, data_fields, progressbar=None):
+    movie = IMDbUtils.get_movie(imdb_id)
+
+    for viewing in viewings:
+        if viewing['imdb_id'] == movie.movieID:
+            for field_name in data_fields:
+                if field_name in movie:
+                    viewing[field_name] = movie[field_name]
+
+    if progressbar is not None:
+        progressbar.next()
+
+
+def add_imdb_keywords(imdb_id, viewings, progressbar=None):
+    movie = IMDbUtils.get_movie_keywords(imdb_id)
+
+    for viewing in viewings:
+        if viewing['imdb_id'] == movie.movieID:
+            if 'keywords' in movie:
+                viewing['keywords'] = movie['keywords']
+
+    if progressbar is not None:
+        progressbar.next()
+
+
+def add_imdb_data_to_viewings(viewings, field_names, progress_bar=None):
+    with ThreadPoolExecutor(4) as executor:
+        future_imdb_tasks = set()
+
+        if ('keywords' in field_names and len(field_names) > 1) or ('keywords' not in field_names and len(field_names) > 0):
+            future_imdb_tasks.update(executor.submit(add_imdb_data, viewing['imdb_id'], viewings, field_names, progress_bar) for viewing in viewings)
+
+        if 'keywords' in field_names:
+            future_imdb_tasks.update(executor.submit(add_imdb_keywords, viewing['imdb_id'], viewings, progress_bar) for viewing in viewings)
+
+        progress_bar.max = len(future_imdb_tasks)
+
+    if progress_bar is not None:
+        progress_bar.finish()
+
+
+def filter_viewings(viewings, filter_field):
+    viewings_filtered = {}
+
+    for viewing in viewings:
+        if filter_field in viewing:
+            viewing_field = viewing[filter_field]
+            if isinstance(viewing_field, list):
+                for fve in list(viewing_field):
+                    if fve in viewings_filtered.keys():
+                        viewings_filtered[fve] += [viewing]
+                    else:
+                        viewings_filtered[fve] = [viewing]
+            else:
+                if viewing_field in viewings_filtered.keys():
+                    viewings_filtered[viewing_field] += [viewing]
+                else:
+                    viewings_filtered[viewing_field] = [viewing]
+
+    return viewings_filtered
+
+
+def get_film_list(films):
+    film_links = []
+
+    for film in films:
+        film_link = generate_imdb_film_link(film)
+        film_links.append(film_link)
+
+    if len(film_links) > 0:
+        return "<br>".join(film_links)
+    else:
+        return ""
+
+
+def generate_markdown_link(text, url):
+    return "[{}]({})".format(text, url)
+
+
+def generate_imdb_film_link(film):
+    return generate_markdown_link(film['title'], generate_imdb_url(film['imdb_id']))
+
+
+def generate_wikipedia_page_link(page_title):
+    return generate_markdown_link(page_title, generate_wikipedia_url(page_title))
+
+
+def generate_imdb_url(imdb_id):
+    return "https://www.imdb.com/title/tt{}/".format(imdb_id)
+
+
+def generate_wikipedia_url(page_title):
+    return "https://en.wikipedia.org/wiki/{}".format(page_title.replace(" ", "_"))

wiki_pages/FilmsByCountry.py

@@ -0,0 +1,104 @@
+import base64
+from collections import Counter, OrderedDict
+import csv
+import os
+import pyvips
+import worldmap
+import warnings
+
+from bookstack import Bookstack
+from vcinema_utils import VCinemaUtils
+
+warnings.filterwarnings("ignore")
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-country
+PAGE_ID = 34
+
+
+def get_films_by_country(viewings):
+    viewings_filtered_by_country = VCinemaUtils.filter_viewings(viewings, "countries")
+
+    if "Czechia" in viewings_filtered_by_country.keys():
+        viewings_filtered_by_country["Czech Republic"] = viewings_filtered_by_country["Czechia"]
+        viewings_filtered_by_country.pop("Czechia")
+
+    return viewings_filtered_by_country
+
+
+def update_page(token_id, token_secret, films_by_country):
+    page = build_page(films_by_country)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+
+
+def build_page(films_by_country):
+    table = build_table(films_by_country)
+
+    country_counter = Counter(films_by_country)
+    png_data = draw_map(country_counter)
+    encoded = base64.b64encode(png_data).decode("utf-8")
+    image = "![](data:image/png;base64,{})".format(encoded)
+    page = image + "\n" + table
+
+    return page
+
+
+def get_flags_dict():
+    flags = {}
+
+    with open('country-flags.csv', newline='') as f:
+        reader = csv.reader(f, quotechar="\"")
+        next(reader, None)  # skip the headers
+
+        for row in reader:
+            flags[row[0]] = row[1]
+
+    return flags
+
+
+def build_table(films_by_country):
+    films_by_country_sorted = OrderedDict(sorted(films_by_country.items(), key=lambda t: t[0]))
+
+    flags = get_flags_dict()
+
+    table = "| Country | Films |\n| - | - |"
+
+    for country, films in films_by_country_sorted.items():
+        table += "\n"
+
+        row_data = []
+
+        country_label = country
+        if country in flags.keys():
+            country_label += " "
+            country_label += flags[country]
+
+        row_data.append(country_label)
+        row_data.append(VCinemaUtils.get_film_list(films))
+
+        table += " | ".join(row_data)
+
+    return table
+
+
+def draw_map(films_by_country, file_name="vcinema_map.svg"):
+    films_by_country['Germany'] += films_by_country['West Germany']
+    del films_by_country['West Germany']
+
+    counter = Counter(films_by_country)
+    countries = [k for k, v in counter.items()]
+    counts = [len(v) for _, v in counter.items()]
+
+    max_count = max(counts)
+
+    opacity = [0.5 + (x / (float(max_count))/2.0) for x in counts]
+
+    worldmap.plot(countries, cmap=["#FF4000"], opacity=opacity, filename=file_name, verbose=False)
+
+    image = pyvips.Image.new_from_file(file_name)
+    image = image.thumbnail_image(1000, crop=pyvips.Interesting.ALL)
+
+    png_data = image.write_to_buffer(".png")
+
+    os.remove(file_name)
+
+    return png_data

wiki_pages/FilmsByReference.py

@@ -0,0 +1,100 @@
+from collections import OrderedDict
+import wikipedia
+
+from bookstack import Bookstack
+from vcinema_utils import VCinemaUtils
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/references
+PAGE_ID = 62
+
+
+def get_films_by_reference(viewings):
+    films_by_reference = {}
+
+    for viewing in viewings:
+        if "keywords" in viewing.keys():
+            for keyword in viewing["keywords"]:
+                if keyword.startswith("reference-to-"):
+
+                    for reference in films_by_reference:
+                        if keyword in films_by_reference[reference]["keywords"]:
+                            films_by_reference[reference]["films"].append(viewing)
+                            break
+                    else:
+                        keyword = keyword[13:]
+
+                        if keyword.startswith("a-"):
+                            keyword = keyword[2:]
+
+                        if keyword.endswith("-character"):
+                            keyword = keyword[:-10]
+
+                        referenced = keyword.replace("-", " ")
+
+                        try:
+                            searches = wikipedia.search(referenced, suggestion=False)
+                            referenced_page = wikipedia.page(title=referenced, auto_suggest=False)
+
+                            page_title = referenced_page.title
+                            page_url = referenced_page.url
+
+                        except wikipedia.DisambiguationError as e:
+                            page_title = e.title
+                            page_title = page_title[0].upper() + page_title[1:]
+                            page_url = VCinemaUtils.generate_wikipedia_url(page_title)
+                        except wikipedia.PageError as _:
+                            if len(searches) > 0:
+                                try:
+                                    referenced_page = wikipedia.page(title=searches[0], auto_suggest=False)
+
+                                    page_title = referenced_page.title
+                                    page_url = referenced_page.url
+                                except wikipedia.DisambiguationError as e:
+                                    page_title = e.title
+                                    page_title = page_title[0].upper() + page_title[1:]
+                                    page_url = VCinemaUtils.generate_wikipedia_url(page_title)
+                            else:
+                                page_title = referenced.title()
+                                page_url = None
+
+                        if page_title in films_by_reference.keys():
+                            films_by_reference[page_title]["keywords"].append(keyword)
+
+                            if viewing not in films_by_reference[page_title]["films"]:
+                                films_by_reference[page_title]["films"].append(viewing)
+
+                        else:
+                            films_by_reference[page_title] = {"url": page_url,
+                                                              "keywords": [keyword],
+                                                              "films": [viewing]}
+
+    return films_by_reference
+
+
+def update_page(token_id, token_secret, films_by_reference_keyword):
+    page = build_page(films_by_reference_keyword)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+
+
+def build_page(films_by_reference):
+    films_by_reference = OrderedDict(sorted(films_by_reference.items(), key=lambda t: t[0]))
+
+    table = "| Referenced | Films |\n| - | - |"
+
+    for reference, referenced in films_by_reference.items():
+        table += "\n"
+
+        row_data = []
+
+        reference_url = referenced["url"]
+        referenced_films = referenced["films"]
+
+        if reference_url is None:
+            row_data.append(reference)
+        else:
+            row_data.append(VCinemaUtils.generate_markdown_link(reference, reference_url))
+        row_data.append(VCinemaUtils.get_film_list(referenced_films))
+
+        table += " | ".join(row_data)
+
+    return table

wiki_pages/FilmsByYear.py

@@ -0,0 +1,35 @@
+from collections import OrderedDict
+
+from bookstack import Bookstack
+from vcinema_utils import VCinemaUtils
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-release-year
+PAGE_ID = 24
+
+
+def get_films_by_year(viewings):
+    viewings_filtered_by_year = VCinemaUtils.filter_viewings(viewings, "year")
+
+    return viewings_filtered_by_year
+
+
+def update_page(token_id, token_secret, films_by_year):
+    page = build_page(films_by_year)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+
+
+def build_page(films_by_year):
+    films_by_year_sorted = OrderedDict(sorted(films_by_year.items(), key=lambda t: t[0], reverse=True))
+
+    page = "| Year | Films |\n| - | - |"
+
+    for year in films_by_year_sorted.keys():
+        page += "\n"
+
+        row_data = []
+        row_data.append(str(year))
+        row_data.append(VCinemaUtils.get_film_list(films_by_year_sorted[year]))
+
+        page += " | ".join(row_data)
+
+    return page

wiki_pages/HiddenThemes.py

@@ -0,0 +1,86 @@
+from collections import OrderedDict
+
+from bookstack import Bookstack
+from vcinema_utils import VCinemaUtils
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-reference
+PAGE_ID = 63
+
+
+def get_hidden_themes(viewings, token_id, token_secret):
+    # Bit horrible to need to request this again, but it affects the order of the result table
+    viewings_ungrouped = VCinemaUtils.get_vcinema_viewings(token_id, token_secret, combine_repeat_viewings=False)
+
+    # Copy keywords from grouped viewings to ungrouped viewings
+    for viewing_ungrouped in viewings_ungrouped:
+        for viewing in viewings:
+            if viewing['imdb_id'] == viewing_ungrouped['imdb_id']:
+                if 'keywords' in viewing:
+                    viewing_ungrouped['keywords'] = viewing['keywords']
+                    break
+
+    viewings_filtered_watch_date = VCinemaUtils.filter_viewings(viewings_ungrouped, "date_watched")
+
+    for date, viewings in viewings_filtered_watch_date.items():
+        viewing_dict = {"viewings": viewings}
+
+        viewings_filtered_watch_date[date] = viewing_dict
+
+    # Add hidden themes
+    for date, data in viewings_filtered_watch_date.items():
+        keyword_counts = {}
+
+        if len(data['viewings']) > 1:
+            for viewing in data['viewings']:
+                if 'keywords' in viewing:
+                    for keyword in viewing['keywords']:
+                        if keyword in keyword_counts.keys():
+                            keyword_counts[keyword] += 1
+                        else:
+                            keyword_counts[keyword] = 1
+
+            keyword_counts = {k: v for k, v in sorted(keyword_counts.items(), key=lambda item: item[1], reverse=True)}
+            hidden_themes = {}
+
+            for keyword in keyword_counts:
+                rating = float(keyword_counts[keyword]) / float(len(data['viewings']))
+                if rating > 0.5:
+                    hidden_themes[keyword] = rating
+
+            viewings_filtered_watch_date[date]['hidden_themes'] = hidden_themes
+
+    return viewings_filtered_watch_date
+
+
+def update_page(token_id, token_secret, hidden_themes):
+    page = build_page(hidden_themes)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+
+
+def build_page(hidden_themes):
+    hidden_themes = OrderedDict(sorted(hidden_themes.items(), key=lambda t: t[0]))
+
+    table = "| Date | Films | Hidden Themes |\n| - | - | - |"
+
+    for date, data in hidden_themes.items():
+        table += "\n"
+
+        row_data = []
+        row_data.append(str(date))
+        row_data.append(VCinemaUtils.get_film_list(data['viewings']))
+        if 'hidden_themes' in data and data['hidden_themes'] != {}:
+            hidden_theme_labels = []
+
+            for hidden_theme in sorted(data['hidden_themes'].keys()):
+                if data['hidden_themes'][hidden_theme] == 1:
+                    hidden_theme_labels.append(hidden_theme)
+                else:
+                    hidden_theme_labels.append("<i>{} ({}%)</i>".format(hidden_theme, round(data['hidden_themes'][hidden_theme] * 100)))
+
+            row_data.append("<br>".join(hidden_theme_labels))
+        else:
+            row_data.append("N/A")
+
+        table += " | ".join(row_data)
+
+    return table

wiki_pages/KeywordScores.py

@@ -0,0 +1,81 @@
+from collections import OrderedDict
+from progress.bar import IncrementalBar
+import math
+from concurrent.futures import ThreadPoolExecutor
+
+from bookstack import Bookstack
+from imdb_utils import IMDbUtils
+from vcinema_utils import VCinemaUtils
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/keyword-scores
+PAGE_ID = 23
+
+
+def get_keyword_scores(viewings):
+    viewings_filtered_keyword = VCinemaUtils.filter_viewings(viewings, "keywords")
+
+    for keyword, viewings in viewings_filtered_keyword.items():
+        viewings_filtered_keyword[keyword] = {"vcinema_films": viewings}
+
+    min_vcinema_count = 2
+    min_imdb_count = 4
+
+    add_keyword_totals(viewings_filtered_keyword, min_vcinema_count)
+    add_keyword_scores(viewings_filtered_keyword, min_vcinema_count, min_imdb_count)
+
+    return viewings_filtered_keyword
+
+
+def update_page(token_id, token_secret, keyword_data):
+    page = build_page(keyword_data)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+
+
+def add_keyword_totals(keywords, min_vcinema_count):
+    keyword_count = len([keyword for keyword in keywords.keys() if len(keywords[keyword]['vcinema_films']) >= min_vcinema_count])
+
+    with IncrementalBar(message='%(percent).1f%% - %(eta)ds remaining', max=keyword_count, check_tty=False) as bar:
+        with ThreadPoolExecutor(6) as executor:
+            for keyword, data in keywords.items():
+                if len(data['vcinema_films']) >= min_vcinema_count:
+                    executor.submit(add_keyword_total, keyword, keywords, bar)
+
+
+def add_keyword_total(keyword, keywords, progress_bar=None):
+    keyword_total = IMDbUtils.get_keyword_count(keyword)
+
+    keywords[keyword]['total'] = keyword_total
+
+    if progress_bar is not None:
+        progress_bar.next()
+
+
+def add_keyword_scores(keyword_data, min_vcinema_count, min_imdb_count):
+    for keyword in keyword_data.keys():
+        if 'total' in keyword_data[keyword]:
+            vcinema_count = len(keyword_data[keyword]['vcinema_films'])
+            total_count = keyword_data[keyword]['total']
+
+            if vcinema_count >= min_vcinema_count and total_count >= min_imdb_count:
+                score = vcinema_count / math.log(total_count)
+
+                keyword_data[keyword]['score'] = score
+
+
+def build_page(keyword_data, minimum_score=1.0):
+    keyword_data = {k: v for k, v in keyword_data.items() if 'score' in v and v['score'] >= minimum_score}
+    keyword_data = OrderedDict(sorted(keyword_data.items(), key=lambda t: t[1]['score'], reverse=True))
+
+    table = "| Keyword | Number of VCinema Films | Total IMDb entries | Score |\n| - | - | - | - |"
+
+    for keyword, data in keyword_data.items():
+        table += "\n"
+
+        row_data = []
+        row_data.append(str(keyword))
+        row_data.append(str(len(data['vcinema_films'])))
+        row_data.append(str(data['total']))
+        row_data.append(str(round(data['score'], 3)))
+        table += " | ".join(row_data)
+
+    return table

wiki_pages/ViewingsCsv.py

@@ -0,0 +1,20 @@
+from bs4 import BeautifulSoup
+
+from bookstack import Bookstack
+from vcinema_utils import VCinemaUtils
+
+# Page ID of https://wiki.jacknet.io/books/vcinema/page/csv
+PAGE_ID = 11
+
+
+def update_viewings_csv(token_id, token_secret):
+    print("Retrieving viewings page")
+    html_page = Bookstack.get_page_html(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID)
+
+    soup = BeautifulSoup(html_page, 'html.parser')
+    csv_data = soup.find("code").text.strip().encode('utf-8')
+
+    existing_attachment_id = VCinemaUtils.get_viewings_csv_attachment_id(token_id, token_secret)
+    print("Updating file")
+    Bookstack.update_attachment(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, existing_attachment_id, "vcinema.csv", csv_data, PAGE_ID)
+    print("File updated")

wiki_utils

@@ -1 +0,0 @@
-Subproject commit 88be36d5cd7378a01d9861726bf123715fe81d4a