From 295ea0cde07bbb629be8ed0e447d4f0ebaaf2620 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 27 Nov 2021 11:47:12 +0700 Subject: [PATCH] handling deactivating playlist --- tubearchivist/home/src/download.py | 24 ++++++++++++---- tubearchivist/home/src/index.py | 44 +++++++++++++++++++++++++++--- tubearchivist/home/src/reindex.py | 5 ++++ tubearchivist/home/tasks.py | 3 ++ 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tubearchivist/home/src/download.py b/tubearchivist/home/src/download.py index 742fed2..8280181 100644 --- a/tubearchivist/home/src/download.py +++ b/tubearchivist/home/src/download.py @@ -378,14 +378,17 @@ class PlaylistSubscription: @staticmethod def get_playlists(subscribed_only=True): - """get a list of all playlists""" + """get a list of all active playlists""" data = { "sort": [{"playlist_channel.keyword": {"order": "desc"}}], } + data["query"] = { + "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]} + } if subscribed_only: - data["query"] = {"term": {"playlist_subscribed": {"value": True}}} - else: - data["query"] = {"match_all": {}} + data["query"]["bool"]["must"].append( + {"term": {"playlist_subscribed": {"value": True}}} + ) all_playlists = IndexPaginate("ta_playlist", data).get_results() @@ -482,7 +485,12 @@ class PlaylistSubscription: counter = 1 for playlist_id in all_playlists: size_limit = self.config["subscriptions"]["channel_size"] - playlist = YoutubePlaylist(playlist_id).update_playlist() + playlist_handler = YoutubePlaylist(playlist_id) + playlist = playlist_handler.update_playlist() + if not playlist: + playlist_handler.deactivate() + continue + if size_limit: playlist_entries = playlist["playlist_entries"][:size_limit] else: @@ -735,7 +743,11 @@ class VideoDownloader: playlist_handler = YoutubePlaylist( playlist_id, all_youtube_ids=all_youtube_ids ) - _ = playlist_handler.update_playlist() + playlist_dict = playlist_handler.update_playlist() + if not playlist_dict: + playlist_handler.deactivate() + continue + playlist_handler.add_vids_to_playlist() # notify title = ( diff --git a/tubearchivist/home/src/index.py b/tubearchivist/home/src/index.py index 7185ce7..8664346 100644 --- a/tubearchivist/home/src/index.py +++ b/tubearchivist/home/src/index.py @@ -466,6 +466,8 @@ class YoutubePlaylist: if scrape: playlist_dict = self.get_youtube_playlist() + if not playlist_dict: + return False playlist_dict["playlist_entries"] = self.get_entries() else: playlist_dict = self.get_es_playlist() @@ -474,6 +476,7 @@ class YoutubePlaylist: playlist_dict["playlist_entries"] = self.get_entries() self.playlist_dict = playlist_dict + return True def get_youtube_playlist(self): """get meta data dict from youtube""" @@ -481,12 +484,20 @@ class YoutubePlaylist: obs = { "default_search": "ytsearch", "quiet": True, - "ignoreerrors": True, "skip_download": True, "extract_flat": True, "playlistend": 0, } - playlist = youtube_dl.YoutubeDL(obs).extract_info(url, download=False) + try: + playlist = youtube_dl.YoutubeDL(obs).extract_info( + url, download=False + ) + except ( + youtube_dl.utils.ExtractorError, + youtube_dl.utils.DownloadError, + ): + print("failed to get info for " + self.playlist_id) + return False playlist_es = { "playlist_id": self.playlist_id, @@ -517,14 +528,22 @@ class YoutubePlaylist: obs = { "default_search": "ytsearch", "quiet": True, - "ignoreerrors": True, "skip_download": True, "extract_flat": True, } if playlistend: obs["playlistend"] = playlistend - playlist = youtube_dl.YoutubeDL(obs).extract_info(url, download=False) + try: + playlist = youtube_dl.YoutubeDL(obs).extract_info( + url, download=False + ) + except ( + youtube_dl.utils.ExtractorError, + youtube_dl.utils.DownloadError, + ): + print("failed to get plealist entries for " + self.playlist_id) + return False all_members = [] for idx, entry in enumerate(playlist["entries"]): @@ -596,6 +615,10 @@ class YoutubePlaylist: """update metadata for playlist with data from YouTube""" subscribed = self.get_es_playlist()["playlist_subscribed"] self.get_playlist_dict(scrape=True) + if not self.playlist_dict: + # return false to deactivate + return False + self.playlist_dict["playlist_subscribed"] = subscribed self.upload_to_es() return self.playlist_dict @@ -686,6 +709,19 @@ class YoutubePlaylist: if not response.ok: print(response.text) + def deactivate(self): + """deactivate document on extractor error""" + headers = {"Content-type": "application/json"} + url = f"{self.ES_URL}/ta_playlist/_update/{self.playlist_id}" + data = {"script": "ctx._source.playlist_active = false"} + json_str = json.dumps(data) + response = requests.post( + url, data=json_str, headers=headers, auth=self.ES_AUTH + ) + print(f"deactivated {self.playlist_id}") + if not response.ok: + print(response.text) + class WatchState: """handle watched checkbox for videos and channels""" diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index 5e00ab3..5233a60 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -245,9 +245,14 @@ class Reindex: playlist_id, all_youtube_ids=all_indexed_ids ) playlist = playlist_handler.update_playlist() + if not playlist: + playlist_handler.deactivate() + return + playlist_thumbnail = (playlist_id, playlist["playlist_thumbnail"]) thumb_handler = ThumbManager() thumb_handler.download_playlist([playlist_thumbnail]) + return def reindex(self): """reindex what's needed""" diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 3cda556..dcca887 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -237,6 +237,9 @@ def index_channel_playlists(channel_id): playlist_id, all_youtube_ids=all_youtube_ids ) playlist_handler.get_playlist_dict() + if not playlist_handler.playlist_dict: + # skip if not available + continue # don't add if no videos downloaded downloaded = [ i