use the refactored PendingList class

tubearchivist/home/src/download/queue.py

@@ -106,7 +106,7 @@ class PendingInteract:
 class PendingList(PendingIndex):
     """manage the pending videos list"""
 
-    def __init__(self, youtube_ids):
+    def __init__(self, youtube_ids=False):
         super().__init__()
         self.youtube_ids = youtube_ids
         self.to_skip = False
@@ -173,7 +173,7 @@ class PendingList(PendingIndex):
 
         thumb_handler = ThumbManager()
         for idx, youtube_id in enumerate(self.missing_videos):
-            video_details = self._get_youtube_details(youtube_id)
+            video_details = self.get_youtube_details(youtube_id)
             if not video_details:
                 continue
 
@@ -208,7 +208,7 @@ class PendingList(PendingIndex):
         if idx + 1 % 25 == 0:
             print("adding to queue progress: " + progress)
 
-    def _get_youtube_details(self, youtube_id):
+    def get_youtube_details(self, youtube_id):
         """get details from youtubedl for single pending video"""
         obs = {
             "default_search": "ytsearch",

tubearchivist/home/src/download/subscriptions.py

@@ -61,11 +61,9 @@ class ChannelSubscription:
     def find_missing(self):
         """add missing videos from subscribed channels to pending"""
         all_channels = self.get_channels()
-        pending_handler = queue.PendingList()
+        pending = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
+        pending.get_download()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
+        pending.get_indexed()
-        all_downloaded = pending_handler.get_all_downloaded()
-        to_ignore = all_ids + all_downloaded
 
         missing_videos = []
 
@@ -75,7 +73,7 @@ class ChannelSubscription:
 
             if last_videos:
                 for video in last_videos:
-                    if video[0] not in to_ignore:
+                    if video[0] not in pending.to_skip:
                         missing_videos.append(video[0])
             # notify
             message = {
@@ -129,7 +127,11 @@ class PlaylistSubscription:
 
     def process_url_str(self, new_playlists, subscribed=True):
         """process playlist subscribe form url_str"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        data = {
+            "query": {"match_all": {}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        all_indexed = IndexPaginate("ta_video", data).get_results()
         all_youtube_ids = [i["youtube_id"] for i in all_indexed]
 
         new_thumbs = []
@@ -179,12 +181,11 @@ class PlaylistSubscription:
     @staticmethod
     def get_to_ignore():
         """get all youtube_ids already downloaded or ignored"""
-        pending_handler = queue.PendingList()
+        pending = queue.PendingList()
-        all_pending, all_ignore = pending_handler.get_all_pending()
+        pending.get_download()
-        all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
+        pending.get_indexed()
-        all_downloaded = pending_handler.get_all_downloaded()
+
-        to_ignore = all_ids + all_downloaded
+        return pending.to_skip
-        return to_ignore
 
     def find_missing(self):
         """find videos in subscribed playlists not downloaded yet"""

tubearchivist/home/src/download/thumbnails.py

@@ -58,11 +58,12 @@ class ThumbManager:
     def get_needed_thumbs(self, missing_only=False):
         """get a list of all missing thumbnails"""
         all_thumbs = self.get_all_thumbs()
-        all_indexed = queue.PendingList().get_all_indexed()
+
-        all_in_queue, all_ignored = queue.PendingList().get_all_pending()
+        pending = queue.PendingList()
+        pending.get_indexed()
 
         needed_thumbs = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -71,7 +72,9 @@ class ThumbManager:
             else:
                 needed_thumbs.append((youtube_id, thumb_url))
 
-        for video in all_in_queue + all_ignored:
+        pending.get_download()
+
+        for video in pending.all_pending + pending.all_ignored:
             youtube_id = video["youtube_id"]
             thumb_url = video["vid_thumb_url"]
             if missing_only:
@@ -276,9 +279,11 @@ class ThumbManager:
 
     def get_thumb_list(self):
         """get list of mediafiles and matching thumbnails"""
-        all_indexed = queue.PendingList().get_all_indexed()
+        pending = queue.PendingList()
+        pending.get_indexed()
+
         video_list = []
-        for video in all_indexed:
+        for video in pending.all_videos:
             youtube_id = video["youtube_id"]
             media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
             thumb_path = os.path.join(

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -75,8 +75,9 @@ class VideoDownloader:
             "message": "Scanning your download queue.",
         }
         RedisArchivist().set_message("message:download", mess_dict)
-        all_pending, _ = PendingList().get_all_pending()
+        pending = PendingList()
-        to_add = [i["youtube_id"] for i in all_pending]
+        pending.get_download()
+        to_add = [i["youtube_id"] for i in pending.all_pending]
         if not to_add:
             # there is nothing pending
             print("download queue is empty")
@@ -259,8 +260,9 @@ class VideoDownloader:
         """look for playlist needing to update"""
         print("sync playlists")
         self._add_subscribed_channels()
-        all_indexed = PendingList().get_all_indexed()
+        pending = PendingList()
-        all_youtube_ids = [i["youtube_id"] for i in all_indexed]
+        pending.get_indexed()
+        all_youtube_ids = [i["youtube_id"] for i in pending.all_videos]
         for id_c, channel_id in enumerate(self.channels):
             playlists = YoutubeChannel(channel_id).get_indexed_playlists()
             all_playlist_ids = [i["playlist_id"] for i in playlists]
@@ -302,15 +304,17 @@ class VideoDownloader:
             "query": {"range": {"player.watched_date": {"lte": now_lte}}},
             "sort": [{"player.watched_date": {"order": "asc"}}],
         }
-        all_to_delete = IndexPaginate("ta_video", data).get_results()
+        to_delete = IndexPaginate("ta_video", data).get_results()
-        all_youtube_ids = [i["youtube_id"] for i in all_to_delete]
+        if not to_delete:
-        if not all_youtube_ids:
             return
 
-        for youtube_id in all_youtube_ids:
+        for video in to_delete:
+            youtube_id = video["youtube_id"]
             print(f"autodelete {youtube_id}")
             YoutubeVideo(youtube_id).delete_media_file()
 
         print("add deleted to ignore list")
-        pending_handler = PendingList()
+        vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete]
-        pending_handler.add_to_pending(all_youtube_ids, ignore=True)
+        pending = PendingList(youtube_ids=vids)
+        pending.parse_url_list()
+        pending.add_to_pending(status="ignore")

tubearchivist/home/src/frontend/api_calls.py

@@ -4,7 +4,7 @@ Functionality:
 - called via user input
 """
 
-from home.src.download.queue import PendingList
+from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
     ChannelSubscription,
     PlaylistSubscription,
@@ -110,12 +110,11 @@ class PostData:
 
     def _ignore(self):
         """ignore from download queue"""
-        id_to_ignore = self.exec_val
+        video_id = self.exec_val
-        print("ignore video " + id_to_ignore)
+        print(f"ignore video {video_id}")
-        handler = PendingList()
+        PendingInteract(video_id=video_id, status="ignore")
-        handler.ignore_from_pending([id_to_ignore])
         # also clear from redis queue
-        RedisQueue("dl_queue").clear_item(id_to_ignore)
+        RedisQueue("dl_queue").clear_item(video_id)
         return {"success": True}
 
     @staticmethod
@@ -222,28 +221,25 @@ class PostData:
 
     def _forget_ignore(self):
         """delete from ta_download index"""
-        youtube_id = self.exec_val
+        video_id = self.exec_val
-        print("forgetting from download index: " + youtube_id)
+        print(f"forgetting from download index: {video_id}")
-        PendingList().delete_from_pending(youtube_id)
+        PendingInteract(video_id=video_id).delete_item()
         return {"success": True}
 
     def _add_single(self):
         """add single youtube_id to download queue"""
-        youtube_id = self.exec_val
+        video_id = self.exec_val
-        print("add vid to dl queue: " + youtube_id)
+        print(f"add vid to dl queue: {video_id}")
-        PendingList().delete_from_pending(youtube_id)
+        PendingInteract(video_id=video_id).delete_item()
-        youtube_ids = UrlListParser(youtube_id).process_list()
+        video_ids = UrlListParser(video_id).process_list()
-        extrac_dl.delay(youtube_ids)
+        extrac_dl.delay(video_ids)
         return {"success": True}
 
     def _delete_queue(self):
         """delete download queue"""
         status = self.exec_val
         print("deleting from download queue: " + status)
-        if status == "pending":
+        PendingInteract(status=status).delete_by_status()
-            PendingList().delete_pending("pending")
-        elif status == "ignore":
-            PendingList().delete_pending("ignore")
         return {"success": True}
 
     @staticmethod

tubearchivist/home/src/index/filesystem.py

@@ -59,9 +59,10 @@ class FilesystemScanner:
     def get_all_indexed():
         """get a list of all indexed videos"""
         index_handler = PendingList()
-        all_indexed_raw = index_handler.get_all_indexed()
+        index_handler.get_indexed()
+
         all_indexed = []
-        for video in all_indexed_raw:
+        for video in index_handler.all_videos:
             youtube_id = video["youtube_id"]
             media_url = video["media_url"]
             published = video["published"]

tubearchivist/home/src/index/reindex.py

@@ -266,8 +266,9 @@ class Reindex:
         # playlist
         print(f"reindexing {len(self.all_playlist_ids)} playlists")
         if self.all_playlist_ids:
-            all_indexed = PendingList().get_all_indexed()
+            handler = PendingList()
-            all_indexed_ids = [i["youtube_id"] for i in all_indexed]
+            handler.get_indexed()
+            all_indexed_ids = [i["youtube_id"] for i in handler.all_videos]
             for playlist_id in self.all_playlist_ids:
                 self.reindex_single_playlist(playlist_id, all_indexed_ids)
                 if self.sleep_interval:

tubearchivist/home/tasks.py

@@ -63,9 +63,13 @@ def update_subscribed():
             missing_from_playlists = playlist_handler.find_missing()
             missing = missing_from_channels + missing_from_playlists
             if missing:
-                pending_handler = PendingList()
+                youtube_ids = [
-                all_videos_added = pending_handler.add_to_pending(missing)
+                    {"type": "video", "url": i["youtube_id"]} for i in missing
-                ThumbManager().download_vid(all_videos_added)
+                ]
+                pending_handler = PendingList(youtube_ids=youtube_ids)
+                pending_handler.parse_url_list()
+                pending_handler.add_to_pending()
+
         else:
             print("Did not acquire rescan lock.")
 
@@ -128,19 +132,9 @@ def download_single(youtube_id):
 @shared_task
 def extrac_dl(youtube_ids):
     """parse list passed and add to pending"""
-    pending_handler = PendingList()
+    pending_handler = PendingList(youtube_ids=youtube_ids)
-    missing_videos = pending_handler.parse_url_list(youtube_ids)
+    pending_handler.parse_url_list()
-    all_videos_added = pending_handler.add_to_pending(missing_videos)
+    pending_handler.add_to_pending()
-    missing_playlists = pending_handler.missing_from_playlists
-
-    thumb_handler = ThumbManager()
-    if missing_playlists:
-        new_thumbs = PlaylistSubscription().process_url_str(
-            missing_playlists, subscribed=False
-        )
-        thumb_handler.download_playlist(new_thumbs)
-
-    thumb_handler.download_vid(all_videos_added)
 
 
 @shared_task(name="check_reindex")
@@ -283,8 +277,9 @@ def index_channel_playlists(channel_id):
         print(f"no playlists found for channel {channel_id}")
         return
 
-    all_indexed = PendingList().get_all_indexed()
+    handler = PendingList()
-    all_youtube_ids = [i["youtube_id"] for i in all_indexed]
+    handler.get_indexed()
+    all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
 
     for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
         # notify