diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index e13119b..a308bf4 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -106,7 +106,7 @@ class PendingInteract: class PendingList(PendingIndex): """manage the pending videos list""" - def __init__(self, youtube_ids): + def __init__(self, youtube_ids=False): super().__init__() self.youtube_ids = youtube_ids self.to_skip = False @@ -173,7 +173,7 @@ class PendingList(PendingIndex): thumb_handler = ThumbManager() for idx, youtube_id in enumerate(self.missing_videos): - video_details = self._get_youtube_details(youtube_id) + video_details = self.get_youtube_details(youtube_id) if not video_details: continue @@ -208,7 +208,7 @@ class PendingList(PendingIndex): if idx + 1 % 25 == 0: print("adding to queue progress: " + progress) - def _get_youtube_details(self, youtube_id): + def get_youtube_details(self, youtube_id): """get details from youtubedl for single pending video""" obs = { "default_search": "ytsearch", diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 84627b6..a460af9 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -61,11 +61,9 @@ class ChannelSubscription: def find_missing(self): """add missing videos from subscribed channels to pending""" all_channels = self.get_channels() - pending_handler = queue.PendingList() - all_pending, all_ignore = pending_handler.get_all_pending() - all_ids = [i["youtube_id"] for i in all_ignore + all_pending] - all_downloaded = pending_handler.get_all_downloaded() - to_ignore = all_ids + all_downloaded + pending = queue.PendingList() + pending.get_download() + pending.get_indexed() missing_videos = [] @@ -75,7 +73,7 @@ class ChannelSubscription: if last_videos: for video in last_videos: - if video[0] not in to_ignore: + if video[0] not in pending.to_skip: missing_videos.append(video[0]) # notify message = { @@ -129,7 +127,11 @@ class PlaylistSubscription: def process_url_str(self, new_playlists, subscribed=True): """process playlist subscribe form url_str""" - all_indexed = queue.PendingList().get_all_indexed() + data = { + "query": {"match_all": {}}, + "sort": [{"published": {"order": "desc"}}], + } + all_indexed = IndexPaginate("ta_video", data).get_results() all_youtube_ids = [i["youtube_id"] for i in all_indexed] new_thumbs = [] @@ -179,12 +181,11 @@ class PlaylistSubscription: @staticmethod def get_to_ignore(): """get all youtube_ids already downloaded or ignored""" - pending_handler = queue.PendingList() - all_pending, all_ignore = pending_handler.get_all_pending() - all_ids = [i["youtube_id"] for i in all_ignore + all_pending] - all_downloaded = pending_handler.get_all_downloaded() - to_ignore = all_ids + all_downloaded - return to_ignore + pending = queue.PendingList() + pending.get_download() + pending.get_indexed() + + return pending.to_skip def find_missing(self): """find videos in subscribed playlists not downloaded yet""" diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 2ea8477..fa40b40 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -58,11 +58,12 @@ class ThumbManager: def get_needed_thumbs(self, missing_only=False): """get a list of all missing thumbnails""" all_thumbs = self.get_all_thumbs() - all_indexed = queue.PendingList().get_all_indexed() - all_in_queue, all_ignored = queue.PendingList().get_all_pending() + + pending = queue.PendingList() + pending.get_indexed() needed_thumbs = [] - for video in all_indexed: + for video in pending.all_videos: youtube_id = video["youtube_id"] thumb_url = video["vid_thumb_url"] if missing_only: @@ -71,7 +72,9 @@ class ThumbManager: else: needed_thumbs.append((youtube_id, thumb_url)) - for video in all_in_queue + all_ignored: + pending.get_download() + + for video in pending.all_pending + pending.all_ignored: youtube_id = video["youtube_id"] thumb_url = video["vid_thumb_url"] if missing_only: @@ -276,9 +279,11 @@ class ThumbManager: def get_thumb_list(self): """get list of mediafiles and matching thumbnails""" - all_indexed = queue.PendingList().get_all_indexed() + pending = queue.PendingList() + pending.get_indexed() + video_list = [] - for video in all_indexed: + for video in pending.all_videos: youtube_id = video["youtube_id"] media_url = os.path.join(self.MEDIA_DIR, video["media_url"]) thumb_path = os.path.join( diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index be71049..267e420 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -75,8 +75,9 @@ class VideoDownloader: "message": "Scanning your download queue.", } RedisArchivist().set_message("message:download", mess_dict) - all_pending, _ = PendingList().get_all_pending() - to_add = [i["youtube_id"] for i in all_pending] + pending = PendingList() + pending.get_download() + to_add = [i["youtube_id"] for i in pending.all_pending] if not to_add: # there is nothing pending print("download queue is empty") @@ -259,8 +260,9 @@ class VideoDownloader: """look for playlist needing to update""" print("sync playlists") self._add_subscribed_channels() - all_indexed = PendingList().get_all_indexed() - all_youtube_ids = [i["youtube_id"] for i in all_indexed] + pending = PendingList() + pending.get_indexed() + all_youtube_ids = [i["youtube_id"] for i in pending.all_videos] for id_c, channel_id in enumerate(self.channels): playlists = YoutubeChannel(channel_id).get_indexed_playlists() all_playlist_ids = [i["playlist_id"] for i in playlists] @@ -302,15 +304,17 @@ class VideoDownloader: "query": {"range": {"player.watched_date": {"lte": now_lte}}}, "sort": [{"player.watched_date": {"order": "asc"}}], } - all_to_delete = IndexPaginate("ta_video", data).get_results() - all_youtube_ids = [i["youtube_id"] for i in all_to_delete] - if not all_youtube_ids: + to_delete = IndexPaginate("ta_video", data).get_results() + if not to_delete: return - for youtube_id in all_youtube_ids: + for video in to_delete: + youtube_id = video["youtube_id"] print(f"autodelete {youtube_id}") YoutubeVideo(youtube_id).delete_media_file() print("add deleted to ignore list") - pending_handler = PendingList() - pending_handler.add_to_pending(all_youtube_ids, ignore=True) + vids = [{"type": "video", "url": i["youtube_id"]} for i in to_delete] + pending = PendingList(youtube_ids=vids) + pending.parse_url_list() + pending.add_to_pending(status="ignore") diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index dff7636..16814ba 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -4,7 +4,7 @@ Functionality: - called via user input """ -from home.src.download.queue import PendingList +from home.src.download.queue import PendingInteract from home.src.download.subscriptions import ( ChannelSubscription, PlaylistSubscription, @@ -110,12 +110,11 @@ class PostData: def _ignore(self): """ignore from download queue""" - id_to_ignore = self.exec_val - print("ignore video " + id_to_ignore) - handler = PendingList() - handler.ignore_from_pending([id_to_ignore]) + video_id = self.exec_val + print(f"ignore video {video_id}") + PendingInteract(video_id=video_id, status="ignore") # also clear from redis queue - RedisQueue("dl_queue").clear_item(id_to_ignore) + RedisQueue("dl_queue").clear_item(video_id) return {"success": True} @staticmethod @@ -222,28 +221,25 @@ class PostData: def _forget_ignore(self): """delete from ta_download index""" - youtube_id = self.exec_val - print("forgetting from download index: " + youtube_id) - PendingList().delete_from_pending(youtube_id) + video_id = self.exec_val + print(f"forgetting from download index: {video_id}") + PendingInteract(video_id=video_id).delete_item() return {"success": True} def _add_single(self): """add single youtube_id to download queue""" - youtube_id = self.exec_val - print("add vid to dl queue: " + youtube_id) - PendingList().delete_from_pending(youtube_id) - youtube_ids = UrlListParser(youtube_id).process_list() - extrac_dl.delay(youtube_ids) + video_id = self.exec_val + print(f"add vid to dl queue: {video_id}") + PendingInteract(video_id=video_id).delete_item() + video_ids = UrlListParser(video_id).process_list() + extrac_dl.delay(video_ids) return {"success": True} def _delete_queue(self): """delete download queue""" status = self.exec_val print("deleting from download queue: " + status) - if status == "pending": - PendingList().delete_pending("pending") - elif status == "ignore": - PendingList().delete_pending("ignore") + PendingInteract(status=status).delete_by_status() return {"success": True} @staticmethod diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 0354e83..9601fdc 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -59,9 +59,10 @@ class FilesystemScanner: def get_all_indexed(): """get a list of all indexed videos""" index_handler = PendingList() - all_indexed_raw = index_handler.get_all_indexed() + index_handler.get_indexed() + all_indexed = [] - for video in all_indexed_raw: + for video in index_handler.all_videos: youtube_id = video["youtube_id"] media_url = video["media_url"] published = video["published"] diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 6a0dd4f..b696c0b 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -266,8 +266,9 @@ class Reindex: # playlist print(f"reindexing {len(self.all_playlist_ids)} playlists") if self.all_playlist_ids: - all_indexed = PendingList().get_all_indexed() - all_indexed_ids = [i["youtube_id"] for i in all_indexed] + handler = PendingList() + handler.get_indexed() + all_indexed_ids = [i["youtube_id"] for i in handler.all_videos] for playlist_id in self.all_playlist_ids: self.reindex_single_playlist(playlist_id, all_indexed_ids) if self.sleep_interval: diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 63d84b2..e035b7d 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -63,9 +63,13 @@ def update_subscribed(): missing_from_playlists = playlist_handler.find_missing() missing = missing_from_channels + missing_from_playlists if missing: - pending_handler = PendingList() - all_videos_added = pending_handler.add_to_pending(missing) - ThumbManager().download_vid(all_videos_added) + youtube_ids = [ + {"type": "video", "url": i["youtube_id"]} for i in missing + ] + pending_handler = PendingList(youtube_ids=youtube_ids) + pending_handler.parse_url_list() + pending_handler.add_to_pending() + else: print("Did not acquire rescan lock.") @@ -128,19 +132,9 @@ def download_single(youtube_id): @shared_task def extrac_dl(youtube_ids): """parse list passed and add to pending""" - pending_handler = PendingList() - missing_videos = pending_handler.parse_url_list(youtube_ids) - all_videos_added = pending_handler.add_to_pending(missing_videos) - missing_playlists = pending_handler.missing_from_playlists - - thumb_handler = ThumbManager() - if missing_playlists: - new_thumbs = PlaylistSubscription().process_url_str( - missing_playlists, subscribed=False - ) - thumb_handler.download_playlist(new_thumbs) - - thumb_handler.download_vid(all_videos_added) + pending_handler = PendingList(youtube_ids=youtube_ids) + pending_handler.parse_url_list() + pending_handler.add_to_pending() @shared_task(name="check_reindex") @@ -283,8 +277,9 @@ def index_channel_playlists(channel_id): print(f"no playlists found for channel {channel_id}") return - all_indexed = PendingList().get_all_indexed() - all_youtube_ids = [i["youtube_id"] for i in all_indexed] + handler = PendingList() + handler.get_indexed() + all_youtube_ids = [i["youtube_id"] for i in handler.all_videos] for idx, (playlist_id, playlist_title) in enumerate(all_playlists): # notify