Create get_countries.py

roundup_scripts/get_countries.py

@@ -0,0 +1,36 @@
+import argparse
+from datetime import datetime, timedelta
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+
+
+def get_countries(token_id, token_secret, start_date, end_date):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
+
+    in_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, start_date, end_date)
+
+    in_range_countries = VCinemaUtils.filter_viewings(in_range_viewings, "countries")
+
+    print(f"Countries in range {start_date.strftime('%d-%m-%Y')} - {end_date.strftime('%d-%m-%Y')}:")
+    for country, films in in_range_countries.items():
+        print(f"{country}:{len(films)}")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('start_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('end_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_countries(token['token_id'], token['token_secret'], args.start_date, args.end_date)

roundup_scripts/get_film_map_for_date.py

@@ -0,0 +1,44 @@
+import argparse
+from collections import Counter
+from datetime import datetime
+import json
+from progress.bar import IncrementalBar
+
+from vcinema_utils import VCinemaUtils
+from wiki_pages.FilmsByCountry import draw_map
+
+
+def get_film_map_for_date(token_id, token_secret, date, outfile):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
+
+    viewings_before_year = VCinemaUtils.get_viewings_in_date_range(datetime.min, date)
+
+    viewings_before_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_before_year, "countries")
+
+    country_counter = Counter(viewings_before_year_filtered_by_year)
+    png_data = draw_map(country_counter)
+
+    print("saving map to:" + outfile)
+
+    with open(outfile, "wb") as f:
+        f.write(png_data)
+
+    print("saved")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('outfile', type=str)
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_film_map_for_date(token['token_id'], token['token_secret'], args.date, args.outfile)

roundup_scripts/get_new_countries.py

@@ -1,40 +1,39 @@
 import argparse
-from datetime import datetime
+from datetime import datetime, timedelta
 import json
 from progress.bar import IncrementalBar
 
 from vcinema_utils import VCinemaUtils
 
 
-def get_new_years(token_id, token_secret, year):
+def get_new_countries(token_id, token_secret, start_date, end_date):
     print("Getting viewings")
     viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
 
     with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
         VCinemaUtils.add_imdb_data_to_viewings(viewings, ["countries"], bar)
 
-    viewings_before_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year-1))
+    pre_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, datetime.min, start_date - timedelta(days=1))
-    viewings_end_of_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year))
+    in_range_viewings = VCinemaUtils.get_viewings_in_date_range(viewings, start_date, end_date)
 
-    viewings_before_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_before_year, "countries")
+    pre_range_countries = set(list(VCinemaUtils.filter_viewings(pre_range_viewings, "countries").keys()))
-    viewings_end_of_year_filtered_by_year = VCinemaUtils.filter_viewings(viewings_end_of_year, "countries")
+    in_range_countries = set(list(VCinemaUtils.filter_viewings(in_range_viewings, "countries").keys()))
 
-    years_before = set(list(viewings_before_year_filtered_by_year.keys()))
+    new_countries = in_range_countries - pre_range_countries
-    years_after = set(list(viewings_end_of_year_filtered_by_year.keys()))
 
-    years_diff = years_after - years_before
+    print(f"New countries in range {start_date.strftime('%d-%m-%Y')} - {end_date.strftime('%d-%m-%Y')}:")
-
+    print("\n".join(new_countries))
-    print(years_diff)
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Update wiki pages.')
+    parser = argparse.ArgumentParser()
 
-    parser.add_argument('year', type=int)
+    parser.add_argument('start_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
+    parser.add_argument('end_date', type=lambda s: datetime.strptime(s, '%d-%m-%Y'))
 
     args = parser.parse_args()
 
     with open('../token.json') as json_file:
         token = json.load(json_file)
 
-    get_new_years(token['token_id'], token['token_secret'], args.year)
+    get_new_countries(token['token_id'], token['token_secret'], args.start_date, args.end_date)

roundup_scripts/get_start_of_year_top_10_keywords.py

@@ -0,0 +1,53 @@
+import argparse
+from collections import OrderedDict
+from datetime import datetime
+import json
+from progress.bar import IncrementalBar
+
+from wiki_pages import KeywordScores
+from vcinema_utils import VCinemaUtils
+
+
+def get_new_years(token_id, token_secret, year):
+    print("Getting viewings")
+    viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
+
+    with IncrementalBar('Retrieving movie data', max=len(viewings), suffix='%(percent).1f%% - %(eta)ds remaining', check_tty=False) as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ["keywords"], bar)
+
+    viewings_before_year = VCinemaUtils.get_viewings_before_date(viewings, datetime(day=31, month=12, year=year-1))
+
+    scores = KeywordScores.get_keyword_scores(viewings_before_year)
+
+    keyword_data = {k: v for k, v in scores.items() if 'score' in v and v['score'] >= 1.0}
+    keyword_data = OrderedDict(sorted(keyword_data.items(), key=lambda t: t[1]['score'], reverse=True))
+
+
+    table = "| Keyword | Number of VCinema Films | Total IMDb entries | Score |\n| - | - | - | - |"
+
+    for keyword, data in keyword_data.items():
+        table += "\n"
+
+        row_data = []
+        row_data.append(str(keyword))
+        row_data.append(str(len(data['vcinema_films'])))
+        row_data.append(str(data['total']))
+        row_data.append(str(round(data['score'], 3)))
+        table += " | ".join(row_data)
+
+    # return table
+
+    print(scores)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Update wiki pages.')
+
+    parser.add_argument('year', type=int)
+
+    args = parser.parse_args()
+
+    with open('../token.json') as json_file:
+        token = json.load(json_file)
+
+    get_new_years(token['token_id'], token['token_secret'], args.year)

vcinema_utils/VCinemaUtils.py

@@ -120,14 +120,21 @@ def filter_viewings(viewings, filter_field):
     return viewings_filtered
 
 
-def get_viewings_before_date(viewings, date):
+def get_viewings_in_date_range(viewings, start_date, end_date):
-    viewings_before_date = []
+    viewings_in_date_range = []
 
     for viewing in viewings:
-        if all(view_date["date_watched"] < date for view_date in viewing["viewings"]):
+        in_date_range = []
-            viewings_before_date.append(viewing)
 
-    return viewings_before_date
+        for film_viewing in viewing["viewings"]:
+            if start_date <= film_viewing["date_watched"] <= end_date:
+                in_date_range.append(film_viewing)
+
+        if len(in_date_range) > 0:
+            viewing["viewings"] = in_date_range
+            viewings_in_date_range.append(viewing)
+
+    return viewings_in_date_range
 
 
 def get_film_list(films):