add_img_text.py

@@ -1,26 +0,0 @@
-from wiki_pages import FilmsByCountry
-from vcinema_utils import VCinemaUtils
-
-import argparse
-from collections import OrderedDict
-import imageio
-from progress.bar import IncrementalBar
-from pygifsicle import optimize
-from PIL import Image, ImageFont, ImageDraw, ImageFont
-import io
-
-with open("test.png", "rb") as f:
-    img_bytes = f.read()
-
-stream = io.BytesIO(img_bytes)
-img = Image.open(stream)
-
-print(img.height)
-
-map_editable = ImageDraw.Draw(img)
-
-font = ImageFont.truetype("/System/Library/Fonts/Supplemental/Arial.ttf", 48)
-
-map_editable.text((0, 655-50), "{}".format("2022-04-88"), (255, 64, 0), font=font)
-
-img.show()

country_count.csv

@@ -1,32 +0,0 @@
-Country,Count,Alpha
-Hong Kong,8,HKG
-Canada,15,CAN
-United Kingdom,20,GBR
-United States,147,USA
-Germany,9,DEU
-Russia,3,RUS
-Czechia,1,CZE
-Philippines,2,PHL
-New Zealand,4,NZL
-Australia,5,AUS
-Japan,7,JPN
-Italy,4,ITA
-Uganda,2,UGA
-France,4,FRA
-South Korea,2,KOR
-South Africa,1,ZAF
-North Korea,1,PRK
-Taiwan,1,TWN
-Belgium,2,BEL
-Luxembourg,1,LUX
-China,3,CHN
-Turkey,2,TUR
-Israel,1,ISR
-Nigeria,1,NGA
-Mexico,1,MEX
-Spain,1,ESP
-Thailand,2,THA
-Vietnam,1,VNM
-Kazakhstan,1,KAZ
-Ireland,1,IRL
-India,1,IND

map_test.py

@@ -1,13 +0,0 @@
-import plotly.express as px
-import pycountry
-
-df = px.data.gapminder().query("year==2007")
-
-print(df.head())
-
-fig = px.choropleth(df, locations="iso_alpha",
-                    color="lifeExp", # lifeExp is a column of gapminder
-                    hover_name="country", # column to add to hover information
-                    color_continuous_scale=px.colors.sequential.Plasma)
-
-fig.show()

plot_countries.py

@@ -1,76 +0,0 @@
-from bookstack import Bookstack
-from update_films_by_country import get_films_by_country, build_page as build_films_by_country_page, FILMS_BY_COUNTRY_PAGE_ID
-from update_films_by_year import get_films_by_year, build_page as build_films_by_year_page, FILMS_BY_YEAR_PAGE_ID
-from vcinema_utils import VCinemaUtils
-from update_viewings_csv import update_viewings_csv
-
-import argparse
-from progress.bar import IncrementalBar
-import plotly.express as px
-import pandas
-import geopandas
-import numpy as np
-
-from urllib.request import urlopen
-import json
-
-
-country_alpha3 = {}
-
-
-
-def plot_countries(token_id, token_secret):
-    df = pandas.read_csv('country_count.csv', header=0)
-
-    url = 'https://raw.githubusercontent.com/datasets/geo-countries/master/data/countries.geojson'
-    with urlopen(url) as response:
-        countries = json.load(response)
-
-
-    print(countries['features'][0]['properties'].keys())
-
-    fig = px.choropleth_mapbox(df,geojson=countries,locations=df['Alpha'],color=np.log10(df['Count']),hover_name=df['Country'],hover_data=[df['Count']],color_continuous_scale=px.colors.sequential.YlGn, featureidkey="properties.ISO_A3",mapbox_style="stamen-terrain",opacity=0.5)
-    fig.update_geos(fitbounds="locations")
-    fig.update_traces(hovertemplate="<b>%{hovertext}</b><br>Count: %{customdata}")
-    fig.update_layout(coloraxis_showscale=False)
-
-    fig.show()
-#
-
-def test():
-    from urllib.request import urlopen
-    import json
-    with urlopen('https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json') as response:
-        counties = json.load(response)
-
-    import pandas as pd
-    df = pd.read_csv("https://raw.githubusercontent.com/plotly/datasets/master/fips-unemp-16.csv",
-                     dtype={"fips": str})
-
-    import plotly.graph_objects as go
-
-    print("DF")
-
-    print(df)
-
-    fig = go.Figure(go.Choroplethmapbox(geojson=counties, locations=df.fips, z=df.unemp,
-                                        colorscale="Viridis", zmin=0, zmax=12,
-                                        marker_opacity=0.5, marker_line_width=0))
-    fig.update_layout(mapbox_style="carto-positron",
-                      mapbox_zoom=3, mapbox_center={"lat": 37.0902, "lon": -95.7129})
-    fig.update_layout(margin={"r": 0, "t": 0, "l": 0, "b": 0})
-    fig.show()
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Update wiki pages.')
-    parser.add_argument('token_id', help='API token ID.')
-    parser.add_argument('token_secret', help='API token secret.')
-
-    args = parser.parse_args()
-
-    # test()
-
-    plot_countries(args.token_id, args.token_secret)
-
-# pycountry.countries.get(name='American Samoa').alpha2

token.json

@@ -1 +0,0 @@
-{"token_id":  "4Xr3a4mOmuFBDK9nOuzBOjCfO5n4mWXp", "token_secret": "kSA3Td1pKFpBYtf5K0quCBqeIS61Fjzt"}

update_wiki.py

@@ -2,7 +2,6 @@ from wiki_pages import FilmsByCountry, FilmsByReference, FilmsByYear, HiddenThem
 from vcinema_utils import VCinemaUtils
 
 import argparse
-import json
 from progress.bar import IncrementalBar
 
 
@@ -34,36 +33,56 @@ def update_wiki(token_id, token_secret, update_csv, pages):
     with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar)
 
-    print("Processing viewing data")
+    update_page_count = sum([update_films_by_year, update_films_by_country, update_film_references,
+                             update_hidden_themes, update_keyword_scores])
 
+    with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         if update_films_by_year:
             films_by_year = FilmsByYear.get_films_by_year(viewings)
-        FilmsByYear.update_page(token_id, token_secret, films_by_year)
+            bar.next()
         if update_films_by_country:
             films_by_country = FilmsByCountry.get_films_by_country(viewings)
-        FilmsByCountry.update_page(token_id, token_secret, films_by_country)
+            bar.next()
         if update_film_references:
             films_by_reference = FilmsByReference.get_films_by_reference(viewings)
-        FilmsByReference.update_page(token_id, token_secret, films_by_reference)
+            bar.next()
         if update_hidden_themes:
             hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret)
-        HiddenThemes.update_page(token_id, token_secret, hidden_themes)
+            bar.next()
         if update_keyword_scores:
             keyword_scores = KeywordScores.get_keyword_scores(viewings)
-        KeywordScores.update_page(token_id, token_secret, keyword_scores)
+            bar.next()
 
-    print("Done!")
+        bar.finish()
+
+    with IncrementalBar('Updating pages', max=update_page_count, check_tty=False) as bar:
+        if update_films_by_year:
+            FilmsByYear.update_page(token_id, token_secret, films_by_year)
+            bar.next()
+        if update_films_by_country:
+            FilmsByCountry.update_page(token_id, token_secret, films_by_country)
+            bar.next()
+        if update_film_references:
+            FilmsByReference.update_page(token_id, token_secret, films_by_reference)
+            bar.next()
+        if update_hidden_themes:
+            HiddenThemes.update_page(token_id, token_secret, hidden_themes)
+            bar.next()
+        if update_keyword_scores:
+            KeywordScores.update_page(token_id, token_secret, keyword_scores)
+            bar.next()
+
+        bar.finish()
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Update wiki pages.')
+    parser.add_argument('token_id', help='API token ID.')
+    parser.add_argument('token_secret', help='API token secret.')
 
-    parser.add_argument("--do_not_update_csv", action="store_true")
+    parser.add_argument('--update_csv', help='Update viewings.csv file, default: True', default=True, required=False)
     parser.add_argument('--pages', nargs="+", default=['years', 'countries', 'references', 'themes', 'scores'], required=False)
 
     args = parser.parse_args()
 
-    with open('token.json') as json_file:
+    update_wiki(args.token_id, args.token_secret, args.update_csv, args.pages)
-        token = json.load(json_file)
-
-    update_wiki(token['token_id'], token['token_secret'], not args.do_not_update_csv, args.pages)

wiki_pages/ActorScores.py

@@ -1,26 +0,0 @@
-from collections import Counter, OrderedDict
-import warnings
-
-from vcinema_utils import VCinemaUtils
-
-warnings.filterwarnings("ignore")
-
-
-def get_actor_scores(viewings):
-    for viewing_ind in range(len(viewings)):
-        viewing = viewings[viewing_ind]
-
-        cast_list = []
-
-        for cast_member in viewing['cast']:
-            cast_member_name = (cast_member['name'], cast_member.currentRole)
-            cast_list.append(cast_member_name)
-
-        viewings[viewing_ind]['cast'] = cast_list
-
-    viewings_filtered_by_cast_member = VCinemaUtils.filter_viewings(viewings, "cast")
-
-    viewings_filtered_by_cast_member = OrderedDict(sorted(viewings_filtered_by_cast_member.items(), key=lambda t: len(t[1]), reverse=True))
-
-    return viewings_filtered_by_cast_member
-

wiki_pages/HiddenThemes.py

@@ -28,28 +28,27 @@ def get_hidden_themes(viewings, token_id, token_secret):
 
     # Add hidden themes
     for date, data in viewings_filtered_watch_date.items():
-        keyword_counts = {}
+        hidden_themes = set()
 
         if len(data['viewings']) > 1:
+            viewings_keywords = []
+
             for viewing in data['viewings']:
                 if 'keywords' in viewing:
-                    for keyword in viewing['keywords']:
+                    viewings_keywords.append(set(viewing['keywords']))
-                        if keyword in keyword_counts.keys():
-                            keyword_counts[keyword] += 1
-                        else:
-                            keyword_counts[keyword] = 1
 
-            keyword_counts = {k: v for k, v in sorted(keyword_counts.items(), key=lambda item: item[1], reverse=True)}
+            if len(viewings_keywords) > 1:
+                hidden_themes = set.intersection(*viewings_keywords)
 
-            viewings_filtered_watch_date[date]['keyword_counts'] = keyword_counts
+        if hidden_themes != set():
+            viewings_filtered_watch_date[date]['hidden_themes'] = list(hidden_themes)
 
     return viewings_filtered_watch_date
 
 
 def update_page(token_id, token_secret, hidden_themes):
     page = build_page(hidden_themes)
-    print(page)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
-    # Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
 
 
 def build_page(hidden_themes):