remove imports

bookstack

@@ -1 +1 @@
-Subproject commit 50617e94e83f2d00f96bc2dfc0ffb5ec46dc1fc6
+Subproject commit 5d2e08eabc8186848630f1aa790c72260b9751b0

update_films_by_year_page.py

@@ -1,19 +1,24 @@
 import argparse
 from collections import OrderedDict
+from progress.bar import Bar
 
 from bookstack import Bookstack
 from vcinema_utils import VCinemaUtils
 
 
-def build_table(films_by_year):
+def build_table(films_by_year, progressbar=None):
-    fby_sorted = OrderedDict(sorted(films_by_year.items(), key=lambda t: t[0], reverse=True))
+    films_by_year_sorted = OrderedDict(sorted(films_by_year.items(), key=lambda t: t[0], reverse=True))
 
     page_table = "| Year | Films |\n| - | - |\n"
-    for year in fby_sorted.keys():
+
+    for year in films_by_year_sorted.keys():
         page_table += str(year) + " | "
-        page_table += "<br>".join("[{}](https://www.imdb.com/title/tt{}/)".format(film['title'], film['imdb_id']) for film in fby_sorted[year])
+        page_table += "<br>".join("[{}](https://www.imdb.com/title/tt{}/)".format(film['title'], film['imdb_id']) for film in films_by_year_sorted[year])
         page_table += "\n"
 
+        if progressbar is not None:
+            progressbar.next()
+
     return page_table
 
 
@@ -21,20 +26,22 @@ def update_films_by_year_page(token_id, token_secret):
     print("Retrieving VCinema viewings")
     viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
 
-    print("Retrieving movie data")
+    viewing_count = len(viewings)
-    VCinemaUtils.add_imdb_data_to_viewings(viewings, 'year')
+    with Bar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
+        VCinemaUtils.add_imdb_data_to_viewings(viewings, ['year'], bar)
 
-    print("Processing viewing data")
+    with Bar('Processing viewing data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
-    viewings_by_year = VCinemaUtils.filter_viewings(viewings, 'year')
+        viewings_by_year = VCinemaUtils.filter_viewings(viewings, 'year', bar)
 
-    print("Generating table")
+    year_count = len(viewings_by_year)
-    film_by_year_table = build_table(viewings_by_year)
+    with Bar('Generating table', max=year_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
+        film_by_year_table = build_table(viewings_by_year, bar)
 
     # Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-release-year
-    page_id = "24"
+    page_id = 24
 
     print("Updating page")
-    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, page_id, film_by_year_table)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, page_id, markdown=film_by_year_table)
 
     print("Done!")
 

vcinema_utils/VCinemaUtils.py

@@ -1,14 +1,16 @@
-from progress.bar import Bar
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import functools
 
 from imdb_utils import IMDbUtils
-from wiki_utils import WikiUtils
+from bookstack import Bookstack
 
 
 JACKNET_WIKI_URL = "https://wiki.jacknet.io"
 
 
 def get_viewings_csv_attachment_id(token_id, token_secret):
-    attachments = WikiUtils.get_attachments(JACKNET_WIKI_URL, token_id, token_secret)
+    attachments = Bookstack.get_attachments(JACKNET_WIKI_URL, token_id, token_secret)
 
     # Page ID of "https://wiki.jacknet.io/books/vcinema/page/csv"
     page_id = 11
@@ -17,43 +19,76 @@ def get_viewings_csv_attachment_id(token_id, token_secret):
     return next((x['id'] for x in attachments if x['uploaded_to'] == page_id and x['name'] == viewings_csv_file_name), None)
 
 
-def get_vcinema_viewings(token_id, token_secret):
+def get_vcinema_viewings(token_id, token_secret, combine_repeat_viewings=True):
     attachment_id = get_viewings_csv_attachment_id(token_id, token_secret)
 
-    viewings_csv = WikiUtils.get_attachment_contents(attachment_id, JACKNET_WIKI_URL, token_id, token_secret)
+    viewings_csv = Bookstack.get_attachment(JACKNET_WIKI_URL, token_id, token_secret, attachment_id)
     viewings_csv = viewings_csv.decode("utf-8")
     viewings_csv_rows = viewings_csv.strip().split("\n")
 
     headers = viewings_csv_rows.pop(0).split(",")
     viewings = [dict(zip(headers, row.split(","))) for row in viewings_csv_rows]
 
+    if combine_repeat_viewings:
+        watch_counts = Counter([x['imdb_id'] for x in viewings])
+        repeat_watches = [k for k, v in watch_counts.items() if v > 1]
+
+        for film in repeat_watches:
+            viewing_indexes = [index for index, viewing in enumerate(viewings) if viewing['imdb_id'] == film]
+
+            first_watch = viewings[viewing_indexes[0]]
+            first_watch['date_watched'] = [first_watch['date_watched']]
+
+            for index in viewing_indexes[1::]:
+                first_watch['date_watched'].append(viewings[index]['date_watched'])
+
+            for index in reversed(viewing_indexes[1::]):
+                viewings.pop(index)
+
     return viewings
 
 
-def add_imdb_data_to_viewings(viewings, field_name):
+def increment_progressbar(bar, _):
-    viewing_count = len(viewings)
-
-    with Bar('Processing', max=viewing_count) as bar:
-        bar.message = "Processing"
-        bar.suffix = '%(percent).1f%% - %(eta)ds'
-
-        for (viewing_num, viewing) in enumerate(viewings):
-            imdb_entry = IMDbUtils.get_movie(viewing['imdb_id'])
-
-            viewing[field_name] = imdb_entry[field_name]
     bar.next()
-        bar.finish()
 
 
-def filter_viewings(viewings, filter_field, remove_duplicates=True):
+def add_imdb_data_to_viewings(viewings, field_names, progressbar=None):
+    with ThreadPoolExecutor(4) as executor:
+        future_to_url = {executor.submit(IMDbUtils.get_movie, viewing['imdb_id']) for viewing in viewings}
+
+        if progressbar is not None:
+            for this_future in future_to_url:
+                this_future.add_done_callback(functools.partial(increment_progressbar, progressbar))
+
+        for future in as_completed(future_to_url):
+            imdb_data = future.result()
+
+            for viewing in viewings:
+                if viewing['imdb_id'] == imdb_data.movieID:
+                    for field_name in field_names:
+                        if field_name in imdb_data:
+                            viewing[field_name] = imdb_data[field_name]
+
+
+def filter_viewings(viewings, filter_field, progressbar=None):
     viewings_filtered = {}
 
     for viewing in viewings:
+        if filter_field in viewing:
             viewing_field = viewing[filter_field]
+            if isinstance(viewing_field, list):
+                for fve in list(viewing_field):
+                    if fve in viewings_filtered.keys():
+                        viewings_filtered[fve] += [viewing]
+                    else:
+                        viewings_filtered[fve] = [viewing]
+            else:
                 if viewing_field in viewings_filtered.keys():
-            if not remove_duplicates or not any(x['imdb_id'] == viewing['imdb_id'] for x in viewings_filtered[viewing_field]):
                     viewings_filtered[viewing_field] += [viewing]
                 else:
-            viewings_filtered[viewing[filter_field]] = [viewing]
+                    viewings_filtered[viewing_field] = [viewing]
+
+        if progressbar is not None:
+            progressbar.next()
 
     return viewings_filtered