From d0e0903d48fc020feb9dce228834f254bf11051f Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 18 Nov 2021 21:50:35 +0700 Subject: [PATCH] use remote data for playlist find_missing --- tubearchivist/home/src/download.py | 17 +++++++++++++---- tubearchivist/home/src/index.py | 8 ++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/src/download.py b/tubearchivist/home/src/download.py index 2b0a20b..8674428 100644 --- a/tubearchivist/home/src/download.py +++ b/tubearchivist/home/src/download.py @@ -402,19 +402,26 @@ class PlaylistSubscription: return True - def find_missing(self): - """find videos in subscribed playlists not downloaded yet""" - all_playlists = self.get_playlists() + @staticmethod + def get_to_ignore(): + """get all youtube_ids already downloaded or ignored""" pending_handler = PendingList() all_pending, all_ignore = pending_handler.get_all_pending() all_ids = [i["youtube_id"] for i in all_ignore + all_pending] all_downloaded = pending_handler.get_all_downloaded() to_ignore = all_ids + all_downloaded + return to_ignore + + def find_missing(self): + """find videos in subscribed playlists not downloaded yet""" + all_playlists = [i["playlist_id"] for i in self.get_playlists()] + to_ignore = self.get_to_ignore() missing_videos = [] counter = 1 - for playlist in all_playlists: + for playlist_id in all_playlists: size_limit = self.config["subscriptions"]["channel_size"] + playlist = YoutubePlaylist(playlist_id).update_playlist() if size_limit: playlist_entries = playlist["playlist_entries"][:size_limit] else: @@ -448,6 +455,7 @@ class VideoDownloader: def __init__(self, youtube_id_list=False): self.youtube_id_list = youtube_id_list self.config = AppConfig().config + self.channels = set() def run_queue(self): """setup download queue in redis loop until no more items""" @@ -468,6 +476,7 @@ class VideoDownloader: print("failed to download " + youtube_id) continue vid_dict = index_new_video(youtube_id) + self.channels.add(vid_dict["channel"]["channel_id"]) self.move_to_archive(vid_dict) self.delete_from_pending(youtube_id) diff --git a/tubearchivist/home/src/index.py b/tubearchivist/home/src/index.py index 350a2c3..6e93d3f 100644 --- a/tubearchivist/home/src/index.py +++ b/tubearchivist/home/src/index.py @@ -588,6 +588,14 @@ class YoutubePlaylist: if not response.ok: print(response.text) + def update_playlist(self): + """update metadata for playlist with data from YouTube""" + subscribed = self.get_es_playlist()["playlist_subscribed"] + self.get_playlist_dict(scrape=True) + self.playlist_dict["playlist_subscribed"] = subscribed + self.upload_to_es() + return self.playlist_dict + def build_nav(self, youtube_id): """find next and previous in playlist of a given youtube_id""" all_entries_available = self.playlist_dict["playlist_entries"]