handling deactivating playlist

tubearchivist/home/src/download.py

@@ -378,14 +378,17 @@ class PlaylistSubscription:
 
     @staticmethod
     def get_playlists(subscribed_only=True):
-        """get a list of all playlists"""
+        """get a list of all active playlists"""
         data = {
             "sort": [{"playlist_channel.keyword": {"order": "desc"}}],
         }
+        data["query"] = {
+            "bool": {"must": [{"term": {"playlist_active": {"value": True}}}]}
+        }
         if subscribed_only:
-            data["query"] = {"term": {"playlist_subscribed": {"value": True}}}
+            data["query"]["bool"]["must"].append(
-        else:
+                {"term": {"playlist_subscribed": {"value": True}}}
-            data["query"] = {"match_all": {}}
+            )
 
         all_playlists = IndexPaginate("ta_playlist", data).get_results()
 
@@ -482,7 +485,12 @@ class PlaylistSubscription:
         counter = 1
         for playlist_id in all_playlists:
             size_limit = self.config["subscriptions"]["channel_size"]
-            playlist = YoutubePlaylist(playlist_id).update_playlist()
+            playlist_handler = YoutubePlaylist(playlist_id)
+            playlist = playlist_handler.update_playlist()
+            if not playlist:
+                playlist_handler.deactivate()
+                continue
+
             if size_limit:
                 playlist_entries = playlist["playlist_entries"][:size_limit]
             else:
@@ -735,7 +743,11 @@ class VideoDownloader:
                 playlist_handler = YoutubePlaylist(
                     playlist_id, all_youtube_ids=all_youtube_ids
                 )
-                _ = playlist_handler.update_playlist()
+                playlist_dict = playlist_handler.update_playlist()
+                if not playlist_dict:
+                    playlist_handler.deactivate()
+                    continue
+
                 playlist_handler.add_vids_to_playlist()
                 # notify
                 title = (

tubearchivist/home/src/index.py

@@ -466,6 +466,8 @@ class YoutubePlaylist:
 
         if scrape:
             playlist_dict = self.get_youtube_playlist()
+            if not playlist_dict:
+                return False
             playlist_dict["playlist_entries"] = self.get_entries()
         else:
             playlist_dict = self.get_es_playlist()
@@ -474,6 +476,7 @@ class YoutubePlaylist:
                 playlist_dict["playlist_entries"] = self.get_entries()
 
         self.playlist_dict = playlist_dict
+        return True
 
     def get_youtube_playlist(self):
         """get meta data dict from youtube"""
@@ -481,12 +484,20 @@ class YoutubePlaylist:
         obs = {
             "default_search": "ytsearch",
             "quiet": True,
-            "ignoreerrors": True,
             "skip_download": True,
             "extract_flat": True,
             "playlistend": 0,
         }
-        playlist = youtube_dl.YoutubeDL(obs).extract_info(url, download=False)
+        try:
+            playlist = youtube_dl.YoutubeDL(obs).extract_info(
+                url, download=False
+            )
+        except (
+            youtube_dl.utils.ExtractorError,
+            youtube_dl.utils.DownloadError,
+        ):
+            print("failed to get info for " + self.playlist_id)
+            return False
 
         playlist_es = {
             "playlist_id": self.playlist_id,
@@ -517,14 +528,22 @@ class YoutubePlaylist:
         obs = {
             "default_search": "ytsearch",
             "quiet": True,
-            "ignoreerrors": True,
             "skip_download": True,
             "extract_flat": True,
         }
         if playlistend:
             obs["playlistend"] = playlistend
 
-        playlist = youtube_dl.YoutubeDL(obs).extract_info(url, download=False)
+        try:
+            playlist = youtube_dl.YoutubeDL(obs).extract_info(
+                url, download=False
+            )
+        except (
+            youtube_dl.utils.ExtractorError,
+            youtube_dl.utils.DownloadError,
+        ):
+            print("failed to get plealist entries for " + self.playlist_id)
+            return False
 
         all_members = []
         for idx, entry in enumerate(playlist["entries"]):
@@ -596,6 +615,10 @@ class YoutubePlaylist:
         """update metadata for playlist with data from YouTube"""
         subscribed = self.get_es_playlist()["playlist_subscribed"]
         self.get_playlist_dict(scrape=True)
+        if not self.playlist_dict:
+            # return false to deactivate
+            return False
+
         self.playlist_dict["playlist_subscribed"] = subscribed
         self.upload_to_es()
         return self.playlist_dict
@@ -686,6 +709,19 @@ class YoutubePlaylist:
         if not response.ok:
             print(response.text)
 
+    def deactivate(self):
+        """deactivate document on extractor error"""
+        headers = {"Content-type": "application/json"}
+        url = f"{self.ES_URL}/ta_playlist/_update/{self.playlist_id}"
+        data = {"script": "ctx._source.playlist_active = false"}
+        json_str = json.dumps(data)
+        response = requests.post(
+            url, data=json_str, headers=headers, auth=self.ES_AUTH
+        )
+        print(f"deactivated {self.playlist_id}")
+        if not response.ok:
+            print(response.text)
+
 
 class WatchState:
     """handle watched checkbox for videos and channels"""

tubearchivist/home/src/reindex.py

@@ -245,9 +245,14 @@ class Reindex:
             playlist_id, all_youtube_ids=all_indexed_ids
         )
         playlist = playlist_handler.update_playlist()
+        if not playlist:
+            playlist_handler.deactivate()
+            return
+
         playlist_thumbnail = (playlist_id, playlist["playlist_thumbnail"])
         thumb_handler = ThumbManager()
         thumb_handler.download_playlist([playlist_thumbnail])
+        return
 
     def reindex(self):
         """reindex what's needed"""

tubearchivist/home/tasks.py

@@ -237,6 +237,9 @@ def index_channel_playlists(channel_id):
             playlist_id, all_youtube_ids=all_youtube_ids
         )
         playlist_handler.get_playlist_dict()
+        if not playlist_handler.playlist_dict:
+            # skip if not available
+            continue
         # don't add if no videos downloaded
         downloaded = [
             i