Create get_countries.py

roundup_scripts/get_countries.py

@@ -0,0 +1,36 @@
+import argparse
+from datetime import datetime, timedelta
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+
+
+def get_countries(token_id, token_secret, start_date, end_date):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
+
+    in_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, start_date, end_date)
+
+    in_range_countries = VCinemaUtils.filter_viewings(in_range_viewings, "countries")
+
+    print(f"Countries in range {start_date.strftime('%d-%m-%Y')} - {end_date.strftime('%d-%m-%Y')}:")
+    for country, films in in_range_countries.items():
+        print(f"{country}:{len(films)}")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('start_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('end_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_countries(token['token_id'], token['token_secret'], args.start_date, args.end_date)

roundup_scripts/get_film_map_for_date.py

@@ -0,0 +1,44 @@
+import argparse
+from collections import Counter
+from datetime import datetime
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+from wiki_pages.FilmsByCountry import draw_map
+
+
+def get_film_map_for_date(token_id, token_secret, date, outfile):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
+
+    viewings_before_year = VCinemaUtils.get_viewings_in_date_range(datetime.min, date)
+
+    viewings_before_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_before_year, "countries")
+
+    country_counter = Counter(viewings_before_year_filtered_by_year)
+    png_data = draw_map(country_counter)
+
+    print("saving map to:" + outfile)
+
+    with open(outfile, "wb") as f:
+        f.write(png_data)
+
+    print("saved")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('outfile', type=str)
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_film_map_for_date(token['token_id'], token['token_secret'], args.date, args.outfile)

roundup_scripts/get_new_countries.py

@@ -0,0 +1,39 @@
+import argparse
+from datetime import datetime, timedelta
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+
+
+def get_new_countries(token_id, token_secret, start_date, end_date):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
+
+    pre_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, datetime.min, start_date - timedelta(days=1))
+    in_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, start_date, end_date)
+
+    pre_range_countries = set(list(VCinemaUtils.filter_viewings(pre_range_viewings, "countries").keys()))
+    in_range_countries = set(list(VCinemaUtils.filter_viewings(in_range_viewings, "countries").keys()))
+
+    new_countries = in_range_countries - pre_range_countries
+
+    print(f"New countries in range {start_date.strftime('%d-%m-%Y')} - {end_date.strftime('%d-%m-%Y')}:")
+    print("\n".join(new_countries))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('start_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('end_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_new_countries(token['token_id'], token['token_secret'], args.start_date, args.end_date)

roundup_scripts/get_new_years.py

@@ -0,0 +1,40 @@
+import argparse
+from datetime import datetime
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+
+
+def get_new_years(token_id, token_secret, year):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["year"], bar)
+
+    viewings_before_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year-1))
+    viewings_end_of_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year))
+
+    viewings_before_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_before_year, "year")
+    viewings_end_of_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_end_of_year, "year")
+
+    years_before = set(list(viewings_before_year_filtered_by_year.keys()))
+    years_after = set(list(viewings_end_of_year_filtered_by_year.keys()))
+
+    years_diff = years_after - years_before
+
+    print(years_diff)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update wiki pages.')
+
+    parser.add_argument('year', type=int)
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_new_years(token['token_id'], token['token_secret'], args.year)

roundup_scripts/get_start_of_year_top_10_keywords.py

@@ -0,0 +1,53 @@
+import argparse
+from collections import OrderedDict
+from datetime import datetime
+import json
+from progress.bar import IncrementalBar
+
+from wiki_pages import KeywordScores
+from vcinema_utils import VCinemaUtils
+
+
+def get_new_years(token_id, token_secret, year):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["keywords"], bar)
+
+    viewings_before_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year-1))
+
+    scores = KeywordScores.get_keyword_scores(viewings_before_year)
+
+    keyword_data = {k: v for k, v in scores.items() if 'score' in v and v['score'] >= 1.0}
+    keyword_data = OrderedDict(sorted(keyword_data.items(), key=lambda t: t[1]['score'], reverse=True))
+
+
+    table = "| Keyword | Number of VCinema Films | Total IMDb entries | Score |\n| - | - | - | - |"
+
+    for keyword, data in keyword_data.items():
+        table += "\n"
+
+        row_data = []
+        row_data.append(str(keyword))
+        row_data.append(str(len(data['vcinema_films'])))
+        row_data.append(str(data['total']))
+        row_data.append(str(round(data['score'], 3)))
+        table += " | ".join(row_data)
+
+    # return table
+
+    print(scores)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update wiki pages.')
+
+    parser.add_argument('year', type=int)
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_new_years(token['token_id'], token['token_secret'], args.year)

vcinema_utils/VCinemaUtils.py

@@ -1,6 +1,7 @@
 from collections import Counter
 from concurrent.futures import ThreadPoolExecutor
 import csv
+from datetime import datetime
 
 from imdb_utils import IMDbUtils
 from bookstack import Bookstack
@@ -33,7 +34,9 @@ def get_vcinema_viewings(token_id, token_secret, viewings_csv=None, combine_repe
     if combine_repeat_viewings:
         for viewing in viewings:
             viewing['viewings'] = [
-                {'date_watched': viewing['date_watched'], 'season': viewing['season'], 'rating': viewing['rating']}]
+                {'date_watched': datetime.strptime(viewing['date_watched'], "%Y-%m-%d"),
+                 'season': viewing['season'],
+                 'rating': viewing['rating']}]
             viewing.pop('date_watched')
             viewing.pop('season')
             viewing.pop('rating')
@@ -117,6 +120,23 @@ def filter_viewings(viewings, filter_field):
     return viewings_filtered
 
 
+def get_viewings_in_date_range(viewings, start_date, end_date):
+    viewings_in_date_range = []
+
+    for viewing in viewings:
+        in_date_range = []
+
+        for film_viewing in viewing["viewings"]:
+            if start_date <= film_viewing["date_watched"] <= end_date:
+                in_date_range.append(film_viewing)
+
+        if len(in_date_range) > 0:
+            viewing["viewings"] = in_date_range
+            viewings_in_date_range.append(viewing)
+
+    return viewings_in_date_range
+
+
 def get_film_list(films):
     film_links = []