bookstack

@@ -1 +1 @@
-Subproject commit 5d2e08eabc8186848630f1aa790c72260b9751b0
+Subproject commit 50617e94e83f2d00f96bc2dfc0ffb5ec46dc1fc6

update_films_by_year_page.py

@@ -1,24 +1,19 @@
 import argparse
 from collections import OrderedDict
-from progress.bar import Bar
 
 from bookstack import Bookstack
 from vcinema_utils import VCinemaUtils
 
 
-def build_table(films_by_year, progressbar=None):
-    films_by_year_sorted = OrderedDict(sorted(films_by_year.items(), key=lambda t: t[0], reverse=True))
+def build_table(films_by_year):
+    fby_sorted = OrderedDict(sorted(films_by_year.items(), key=lambda t: t[0], reverse=True))
 
     page_table = "| Year | Films |\n| - | - |\n"
-
-    for year in films_by_year_sorted.keys():
+    for year in fby_sorted.keys():
         page_table += str(year) + " | "
-        page_table += "<br>".join("[{}](https://www.imdb.com/title/tt{}/)".format(film['title'], film['imdb_id']) for film in films_by_year_sorted[year])
+        page_table += "<br>".join("[{}](https://www.imdb.com/title/tt{}/)".format(film['title'], film['imdb_id']) for film in fby_sorted[year])
         page_table += "\n"
 
-        if progressbar is not None:
-            progressbar.next()
-
     return page_table
 
 
@@ -26,22 +21,20 @@ def update_films_by_year_page(token_id, token_secret):
     print("Retrieving VCinema viewings")
     viewings = VCinemaUtils.get_vcinema_viewings(token_id, token_secret)
 
-    viewing_count = len(viewings)
-    with Bar('Retrieving movie data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
-        VCinemaUtils.add_imdb_data_to_viewings(viewings, ['year'], bar)
+    print("Retrieving movie data")
+    VCinemaUtils.add_imdb_data_to_viewings(viewings, 'year')
 
-    with Bar('Processing viewing data', max=viewing_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
-        viewings_by_year = VCinemaUtils.filter_viewings(viewings, 'year', bar)
+    print("Processing viewing data")
+    viewings_by_year = VCinemaUtils.filter_viewings(viewings, 'year')
 
-    year_count = len(viewings_by_year)
-    with Bar('Generating table', max=year_count, suffix='%(percent).1f%% - %(eta)ds remaining') as bar:
-        film_by_year_table = build_table(viewings_by_year, bar)
+    print("Generating table")
+    film_by_year_table = build_table(viewings_by_year)
 
     # Page ID of https://wiki.jacknet.io/books/vcinema/page/films-by-release-year
-    page_id = 24
+    page_id = "24"
 
     print("Updating page")
-    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, page_id, markdown=film_by_year_table)
+    Bookstack.update_page(VCinemaUtils.JACKNET_WIKI_URL, token_id, token_secret, page_id, film_by_year_table)
 
     print("Done!")
 

vcinema_utils/VCinemaUtils.py

@@ -1,16 +1,14 @@
-from collections import Counter
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import functools
+from progress.bar import Bar
 
 from imdb_utils import IMDbUtils
-from bookstack import Bookstack
+from wiki_utils import WikiUtils
 
 
 JACKNET_WIKI_URL = "https://wiki.jacknet.io"
 
 
 def get_viewings_csv_attachment_id(token_id, token_secret):
-    attachments = Bookstack.get_attachments(JACKNET_WIKI_URL, token_id, token_secret)
+    attachments = WikiUtils.get_attachments(JACKNET_WIKI_URL, token_id, token_secret)
 
     # Page ID of "https://wiki.jacknet.io/books/vcinema/page/csv"
     page_id = 11
@@ -19,76 +17,43 @@ def get_viewings_csv_attachment_id(token_id, token_secret):
     return next((x['id'] for x in attachments if x['uploaded_to'] == page_id and x['name'] == viewings_csv_file_name), None)
 
 
-def get_vcinema_viewings(token_id, token_secret, combine_repeat_viewings=True):
+def get_vcinema_viewings(token_id, token_secret):
     attachment_id = get_viewings_csv_attachment_id(token_id, token_secret)
 
-    viewings_csv = Bookstack.get_attachment(JACKNET_WIKI_URL, token_id, token_secret, attachment_id)
+    viewings_csv = WikiUtils.get_attachment_contents(attachment_id, JACKNET_WIKI_URL, token_id, token_secret)
     viewings_csv = viewings_csv.decode("utf-8")
     viewings_csv_rows = viewings_csv.strip().split("\n")
 
     headers = viewings_csv_rows.pop(0).split(",")
     viewings = [dict(zip(headers, row.split(","))) for row in viewings_csv_rows]
 
-    if combine_repeat_viewings:
-        watch_counts = Counter([x['imdb_id'] for x in viewings])
-        repeat_watches = [k for k, v in watch_counts.items() if v > 1]
-
-        for film in repeat_watches:
-            viewing_indexes = [index for index, viewing in enumerate(viewings) if viewing['imdb_id'] == film]
-
-            first_watch = viewings[viewing_indexes[0]]
-            first_watch['date_watched'] = [first_watch['date_watched']]
-
-            for index in viewing_indexes[1::]:
-                first_watch['date_watched'].append(viewings[index]['date_watched'])
-
-            for index in reversed(viewing_indexes[1::]):
-                viewings.pop(index)
-
     return viewings
 
 
-def increment_progressbar(bar, _):
+def add_imdb_data_to_viewings(viewings, field_name):
+    viewing_count = len(viewings)
+
+    with Bar('Processing', max=viewing_count) as bar:
+        bar.message = "Processing"
+        bar.suffix = '%(percent).1f%% - %(eta)ds'
+
+        for (viewing_num, viewing) in enumerate(viewings):
+            imdb_entry = IMDbUtils.get_movie(viewing['imdb_id'])
+
+            viewing[field_name] = imdb_entry[field_name]
             bar.next()
+        bar.finish()
 
 
-def add_imdb_data_to_viewings(viewings, field_names, progressbar=None):
-    with ThreadPoolExecutor(4) as executor:
-        future_to_url = {executor.submit(IMDbUtils.get_movie, viewing['imdb_id']) for viewing in viewings}
-
-        if progressbar is not None:
-            for this_future in future_to_url:
-                this_future.add_done_callback(functools.partial(increment_progressbar, progressbar))
-
-        for future in as_completed(future_to_url):
-            imdb_data = future.result()
-
-            for viewing in viewings:
-                if viewing['imdb_id'] == imdb_data.movieID:
-                    for field_name in field_names:
-                        if field_name in imdb_data:
-                            viewing[field_name] = imdb_data[field_name]
-
-
-def filter_viewings(viewings, filter_field, progressbar=None):
+def filter_viewings(viewings, filter_field, remove_duplicates=True):
     viewings_filtered = {}
 
     for viewing in viewings:
-        if filter_field in viewing:
         viewing_field = viewing[filter_field]
-            if isinstance(viewing_field, list):
-                for fve in list(viewing_field):
-                    if fve in viewings_filtered.keys():
-                        viewings_filtered[fve] += [viewing]
-                    else:
-                        viewings_filtered[fve] = [viewing]
-            else:
         if viewing_field in viewings_filtered.keys():
+            if not remove_duplicates or not any(x['imdb_id'] == viewing['imdb_id'] for x in viewings_filtered[viewing_field]):
                 viewings_filtered[viewing_field] += [viewing]
         else:
-                    viewings_filtered[viewing_field] = [viewing]
-
-        if progressbar is not None:
-            progressbar.next()
+            viewings_filtered[viewing[filter_field]] = [viewing]
 
     return viewings_filtered