rewrite playlist postprocess, ensure refresh, #686

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -40,7 +40,8 @@ class DownloadPostProcess:
         self.pending.get_indexed()
         self.auto_delete_all()
         self.auto_delete_overwrites()
-        self.validate_playlists()
+        to_refresh = self.refresh_playlist()
+        self.match_videos(to_refresh)
         self.get_comments()
 
     def auto_delete_all(self):
@@ -92,48 +93,77 @@ class DownloadPostProcess:
         pending.parse_url_list()
         _ = pending.add_to_pending(status="ignore")
 
-    def validate_playlists(self):
+    def refresh_playlist(self) -> list[str]:
-        """look for playlist needing to update"""
+        """match videos with playlists"""
-        for id_c, channel_id in enumerate(self.download.channels):
+        to_refresh = self._get_to_refresh_playlists()
-            channel = YoutubeChannel(channel_id, task=self.download.task)
+
+        total_playlist = len(to_refresh)
+        for idx, playlist_id in enumerate(to_refresh):
+            playlist = YoutubePlaylist(playlist_id)
+            playlist.update_playlist(skip_on_empty=True)
+
+            if not self.pending.task:
+                continue
+
+            channel_name = playlist.json_data["playlist_channel"]
+            playlist_title = playlist.json_data["playlist_name"]
+            message = [
+                f"Post Processing Playlists for: {channel_name}",
+                f"Validate: {playlist_title} - {idx + 1}/{total_playlist}",
+            ]
+            progress = (idx + 1) / total_playlist
+            self.download.task.send_progress(message, progress=progress)
+
+        return to_refresh
+
+    def _get_to_refresh_playlists(self) -> list[str]:
+        """get playlists to refresh"""
+        if self.download.task:
+            message = ["Post Processing Playlists", "Scanning for Playlists"]
+            self.download.task.send_progress(message)
+
+        to_refresh = []
+        for channel_id in self.download.channels:
+            channel = YoutubeChannel(channel_id)
             overwrites = self.pending.channel_overwrites.get(channel_id, False)
             if overwrites and overwrites.get("index_playlists"):
-                # validate from remote
+                channel.get_all_playlists()
-                channel.index_channel_playlists()
+                to_refresh.extend(channel.all_playlists)
-                continue
 
-            # validate from local
+        subs = PlaylistSubscription().get_playlists()
-            playlists = channel.get_indexed_playlists(active_only=True)
+        for playlist in subs:
-            all_channel_playlist = [i["playlist_id"] for i in playlists]
+            playlist_id = playlist["playlist_id"]
-            self._validate_channel_playlist(all_channel_playlist, id_c)
+            if playlist_id not in to_refresh:
+                to_refresh.append(playlist_id)
 
-    def _validate_channel_playlist(self, all_channel_playlist, id_c):
+        return to_refresh
-        """scan channel for playlist needing update"""
+
-        for id_p, playlist_id in enumerate(all_channel_playlist):
+    def match_videos(self, to_refresh: list[str]) -> None:
+        """scan rest of indexed playlists to match videos"""
+        data = {
+            "query": {
+                "bool": {"must_not": [{"terms": {"playlist_id": to_refresh}}]}
+            },
+            "_source": ["playlist_id"],
+        }
+        playlists = IndexPaginate("ta_playlist", data).get_results()
+        total_playlist = len(playlists)
+        for idx, to_match in enumerate(playlists):
+            playlist_id = to_match["playlist_id"]
             playlist = YoutubePlaylist(playlist_id)
-            playlist.build_json(scrape=True)
+            playlist.get_from_es()
-            if not playlist.json_data:
+            playlist.add_vids_to_playlist()
-                playlist.deactivate()
+            playlist.remove_vids_from_playlist()
+
+            if not self.pending.task:
                 continue
 
-            playlist.add_vids_to_playlist()
+            message = [
-            playlist.upload_to_es()
+                "Post Processing Playlists.",
-            self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
+                f"Validate Playlists: - {idx + 1}/{total_playlist}",
-
+            ]
-    def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
+            progress = (idx + 1) / total_playlist
-        """notify to UI"""
+            self.download.task.send_progress(message, progress=progress)
-        if not self.download.task:
-            return
-
-        total_channel = len(self.download.channels)
-        total_playlist = len(all_channel_playlist)
-
-        message = [
-            f"Post Processing Channels: {id_c}/{total_channel}",
-            f"Validate Playlists {id_p + 1}/{total_playlist}",
-        ]
-        progress = (id_c + 1) / total_channel
-        self.download.task.send_progress(message, progress=progress)
 
     def get_comments(self):
         """get comments from youtube"""