read token from file

add_img_text.py

@@ -0,0 +1,26 @@
+from wiki_pages import FilmsByCountry
+from vcinema_utils import VCinemaUtils
+
+import argparse
+from collections import OrderedDict
+import imageio
+from progress.bar import IncrementalBar
+from pygifsicle import optimize
+from PIL import Image, ImageFont, ImageDraw, ImageFont
+import io
+
+with open("test.png", "rb") as f:
+    img_bytes = f.read()
+
+stream = io.BytesIO(img_bytes)
+img = Image.open(stream)
+
+print(img.height)
+
+map_editable = ImageDraw.Draw(img)
+
+font = ImageFont.truetype("/System/Library/Fonts/Supplemental/Arial.ttf", 48)
+
+map_editable.text((0, 655-50), "{}".format("2022-04-88"), (255, 64, 0), font=font)
+
+img.show()

country_count.csv

@@ -0,0 +1,32 @@
+Country,Count,Alpha
+Hong Kong,8,HKG
+Canada,15,CAN
+United Kingdom,20,GBR
+United States,147,USA
+Germany,9,DEU
+Russia,3,RUS
+Czechia,1,CZE
+Philippines,2,PHL
+New Zealand,4,NZL
+Australia,5,AUS
+Japan,7,JPN
+Italy,4,ITA
+Uganda,2,UGA
+France,4,FRA
+South Korea,2,KOR
+South Africa,1,ZAF
+North Korea,1,PRK
+Taiwan,1,TWN
+Belgium,2,BEL
+Luxembourg,1,LUX
+China,3,CHN
+Turkey,2,TUR
+Israel,1,ISR
+Nigeria,1,NGA
+Mexico,1,MEX
+Spain,1,ESP
+Thailand,2,THA
+Vietnam,1,VNM
+Kazakhstan,1,KAZ
+Ireland,1,IRL
+India,1,IND

map_test.py

@@ -0,0 +1,13 @@
+import plotly.express as px
+import pycountry
+
+df = px.data.gapminder().query("year==2007")
+
+print(df.head())
+
+fig = px.choropleth(df, locations="iso_alpha",
+                    color="lifeExp", # lifeExp is a column of gapminder
+                    hover_name="country", # column to add to hover information
+                    color_continuous_scale=px.colors.sequential.Plasma)
+
+fig.show()

plot_countries.py

@@ -0,0 +1,76 @@
+from bookstack import Bookstack
+from update_films_by_country import get_films_by_country, build_page as build_films_by_country_page, FILMS_BY_COUNTRY_PAGE_ID
+from update_films_by_year import get_films_by_year, build_page as build_films_by_year_page, FILMS_BY_YEAR_PAGE_ID
+from vcinema_utils import VCinemaUtils
+from update_viewings_csv import update_viewings_csv
+
+import argparse
+from progress.bar import IncrementalBar
+import plotly.express as px
+import pandas
+import geopandas
+import numpy as np
+
+from urllib.request import urlopen
+import json
+
+
+country_alpha3 = {}
+
+
+
+def plot_countries(token_id, token_secret):
+    df = pandas.read_csv('country_count.csv', header=0)
+
+    url = 'https://raw.githubusercontent.com/datasets/geo-countries/master/data/countries.geojson'
+    with urlopen(url) as response:
+        countries = json.load(response)
+
+
+    print(countries['features'][0]['properties'].keys())
+
+    fig = px.choropleth_mapbox(df,geojson=countries,locations=df['Alpha'],color=np.log10(df['Count']),hover_name=df['Country'],hover_data=[df['Count']],color_continuous_scale=px.colors.sequential.YlGn, featureidkey="properties.ISO_A3",mapbox_style="stamen-terrain",opacity=0.5)
+    fig.update_geos(fitbounds="locations")
+    fig.update_traces(hovertemplate="<b>%{hovertext}</b><br>Count: %{customdata}")
+    fig.update_layout(coloraxis_showscale=False)
+
+    fig.show()
+#
+
+def test():
+    from urllib.request import urlopen
+    import json
+    with urlopen('https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json') as response:
+        counties = json.load(response)
+
+    import pandas as pd
+    df = pd.read_csv("https://raw.githubusercontent.com/plotly/datasets/master/fips-unemp-16.csv",
+                     dtype={"fips": str})
+
+    import plotly.graph_objects as go
+
+    print("DF")
+
+    print(df)
+
+    fig = go.Figure(go.Choroplethmapbox(geojson=counties, locations=df.fips, z=df.unemp,
+                                        colorscale="Viridis", zmin=0, zmax=12,
+                                        marker_opacity=0.5, marker_line_width=0))
+    fig.update_layout(mapbox_style="carto-positron",
+                      mapbox_zoom=3, mapbox_center={"lat": 37.0902, "lon": -95.7129})
+    fig.update_layout(margin={"r": 0, "t": 0, "l": 0, "b": 0})
+    fig.show()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update wiki pages.')
+    parser.add_argument('token_id', help='API token ID.')
+    parser.add_argument('token_secret', help='API token secret.')
+
+    args = parser.parse_args()
+
+    # test()
+
+    plot_countries(args.token_id, args.token_secret)
+
+# pycountry.countries.get(name='American Samoa').alpha2

token.json

@@ -0,0 +1 @@
+{"token_id":  "4Xr3a4mOmuFBDK9nOuzBOjCfO5n4mWXp", "token_secret": "kSA3Td1pKFpBYtf5K0quCBqeIS61Fjzt"}

update_wiki.py

@@ -2,6 +2,7 @@ from wiki_pages import FilmsByCountry, FilmsByReference, FilmsByYear, HiddenThem
 from vcinema_utils import VCinemaUtils
 
 import argparse
+import json
 from progress.bar import IncrementalBar
 
 
@@ -33,56 +34,36 @@ def update_wiki(token_id, token_secret, update_csv, pages):
     with IncrementalBar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         VCinemaUtils.add_imdb_data_to_viewings(viewings, data_fields, bar)
 
-    update_page_count = sum([update_films_by_year, update_films_by_country, update_film_references,
+    print("Processing viewing data")
-                             update_hidden_themes, update_keyword_scores])
 
-    with IncrementalBar('Processing viewing data', max=update_page_count, suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+    if update_films_by_year:
-        if update_films_by_year:
+        films_by_year = FilmsByYear.get_films_by_year(viewings)
-            films_by_year = FilmsByYear.get_films_by_year(viewings)
+        FilmsByYear.update_page(token_id, token_secret, films_by_year)
-            bar.next()
+    if update_films_by_country:
-        if update_films_by_country:
+        films_by_country = FilmsByCountry.get_films_by_country(viewings)
-            films_by_country = FilmsByCountry.get_films_by_country(viewings)
+        FilmsByCountry.update_page(token_id, token_secret, films_by_country)
-            bar.next()
+    if update_film_references:
-        if update_film_references:
+        films_by_reference = FilmsByReference.get_films_by_reference(viewings)
-            films_by_reference = FilmsByReference.get_films_by_reference(viewings)
+        FilmsByReference.update_page(token_id, token_secret, films_by_reference)
-            bar.next()
+    if update_hidden_themes:
-        if update_hidden_themes:
+        hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret)
-            hidden_themes = HiddenThemes.get_hidden_themes(viewings, token_id, token_secret)
+        HiddenThemes.update_page(token_id, token_secret, hidden_themes)
-            bar.next()
+    if update_keyword_scores:
-        if update_keyword_scores:
+        keyword_scores = KeywordScores.get_keyword_scores(viewings)
-            keyword_scores = KeywordScores.get_keyword_scores(viewings)
+        KeywordScores.update_page(token_id, token_secret, keyword_scores)
-            bar.next()
 
-        bar.finish()
+    print("Done!")
-
-    with IncrementalBar('Updating pages', max=update_page_count, check_tty=False) as bar:
-        if update_films_by_year:
-            FilmsByYear.update_page(token_id, token_secret, films_by_year)
-            bar.next()
-        if update_films_by_country:
-            FilmsByCountry.update_page(token_id, token_secret, films_by_country)
-            bar.next()
-        if update_film_references:
-            FilmsByReference.update_page(token_id, token_secret, films_by_reference)
-            bar.next()
-        if update_hidden_themes:
-            HiddenThemes.update_page(token_id, token_secret, hidden_themes)
-            bar.next()
-        if update_keyword_scores:
-            KeywordScores.update_page(token_id, token_secret, keyword_scores)
-            bar.next()
-
-        bar.finish()
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Update wiki pages.')
-    parser.add_argument('token_id', help='API token ID.')
-    parser.add_argument('token_secret', help='API token secret.')
 
-    parser.add_argument('--update_csv', help='Update viewings.csv file, default: True', default=True, required=False)
+    parser.add_argument("--do_not_update_csv", action="store_true")
     parser.add_argument('--pages', nargs="+", default=['years', 'countries', 'references', 'themes', 'scores'], required=False)
 
     args = parser.parse_args()
 
-    update_wiki(args.token_id, args.token_secret, args.update_csv, args.pages)
+    with open('token.json') as json_file:
+        token = json.load(json_file)
+
+    update_wiki(token['token_id'], token['token_secret'], not args.do_not_update_csv, args.pages)

wiki_pages/ActorScores.py

@@ -0,0 +1,26 @@
+from collections import Counter, OrderedDict
+import warnings
+
+from vcinema_utils import VCinemaUtils
+
+warnings.filterwarnings("ignore")
+
+
+def get_actor_scores(viewings):
+    for viewing_ind in range(len(viewings)):
+        viewing = viewings[viewing_ind]
+
+        cast_list = []
+
+        for cast_member in viewing['cast']:
+            cast_member_name = (cast_member['name'], cast_member.currentRole)
+            cast_list.append(cast_member_name)
+
+        viewings[viewing_ind]['cast'] = cast_list
+
+    viewings_filtered_by_cast_member = VCinemaUtils.filter_viewings(viewings, "cast")
+
+    viewings_filtered_by_cast_member = OrderedDict(sorted(viewings_filtered_by_cast_member.items(), key=lambda t: len(t[1]), reverse=True))
+
+    return viewings_filtered_by_cast_member
+

wiki_pages/HiddenThemes.py

@@ -28,27 +28,28 @@ def get_hidden_themes(viewings, token_id, token_secret):
 
     # Add hidden themes
     for date, data in viewings_filtered_watch_date.items():
-        hidden_themes = set()
+        keyword_counts = {}
 
         if len(data['viewings']) > 1:
-            viewings_keywords = []
-
             for viewing in data['viewings']:
                 if 'keywords' in viewing:
-                    viewings_keywords.append(set(viewing['keywords']))
+                    for keyword in viewing['keywords']:
+                        if keyword in keyword_counts.keys():
+                            keyword_counts[keyword] += 1
+                        else:
+                            keyword_counts[keyword] = 1
 
-            if len(viewings_keywords) > 1:
+            keyword_counts = {k: v for k, v in sorted(keyword_counts.items(), key=lambda item: item[1], reverse=True)}
-                hidden_themes = set.intersection(*viewings_keywords)
 
-        if hidden_themes != set():
+            viewings_filtered_watch_date[date]['keyword_counts'] = keyword_counts
-            viewings_filtered_watch_date[date]['hidden_themes'] = list(hidden_themes)
 
     return viewings_filtered_watch_date
 
 
 def update_page(token_id, token_secret, hidden_themes):
     page = build_page(hidden_themes)
-    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
+    print(page)
+    # Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, PAGE_ID, markdown=page)
 
 
 def build_page(hidden_themes):