From b35440253d107dafe845ae5d600782714743ea2e Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 3 Dec 2021 19:47:43 +0700 Subject: [PATCH] transform add to queue, rescan, and download progress to channel messages --- tubearchivist/home/src/download.py | 66 ++++++++++++---------------- tubearchivist/home/src/reindex.py | 8 ---- tubearchivist/home/src/thumbnails.py | 24 +++++----- tubearchivist/home/tasks.py | 15 ++++--- tubearchivist/home/views.py | 4 +- 5 files changed, 50 insertions(+), 67 deletions(-) diff --git a/tubearchivist/home/src/download.py b/tubearchivist/home/src/download.py index fee1cd0..7632475 100644 --- a/tubearchivist/home/src/download.py +++ b/tubearchivist/home/src/download.py @@ -48,12 +48,12 @@ class PendingList: for entry in youtube_ids: # notify mess_dict = { - "status": "pending", + "status": "message:add", "level": "info", "title": "Adding to download queue.", "message": "Extracting lists", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:add", mess_dict) # extract url = entry["url"] url_type = entry["type"] @@ -123,12 +123,12 @@ class PendingList: # notify progress = f"{counter}/{len(missing_videos)}" mess_dict = { - "status": "pending", + "status": "message:add", "level": "info", "title": "Adding new videos to download queue.", "message": "Progress: " + progress, } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:add", mess_dict) if counter % 25 == 0: print("adding to queue progress: " + progress) counter = counter + 1 @@ -261,13 +261,6 @@ class PendingList: request = requests.post( url, data=query_str, headers=headers, auth=self.ES_AUTH ) - mess_dict = { - "status": "ignore", - "level": "info", - "title": "Added to ignore list", - "message": "", - } - RedisArchivist().set_message("progress:download", mess_dict) if not request.ok: print(request) raise ValueError("failed to set video to ignore") @@ -325,15 +318,13 @@ class ChannelSubscription: for channel in all_channels: channel_id = channel["channel_id"] last_videos = self.get_last_youtube_videos(channel_id) - RedisArchivist().set_message( - "progress:download", - { - "status": "rescan", - "level": "info", - "title": "Scanning channels: Looking for new videos.", - "message": f"Progress: {counter}/{len(all_channels)}", - }, - ) + message = { + "status": "message:rescan", + "level": "info", + "title": "Scanning channels: Looking for new videos.", + "message": f"Progress: {counter}/{len(all_channels)}", + } + RedisArchivist().set_message("message:rescan", message=message) for video in last_videos: youtube_id = video[0] if youtube_id not in to_ignore: @@ -506,15 +497,14 @@ class PlaylistSubscription: playlist_entries = playlist["playlist_entries"] all_missing = [i for i in playlist_entries if not i["downloaded"]] - RedisArchivist().set_message( - "progress:download", - { - "status": "rescan", - "level": "info", - "title": "Scanning playlists: Looking for new videos.", - "message": f"Progress: {counter}/{len(all_playlists)}", - }, - ) + message = { + "status": "message:rescan", + "level": "info", + "title": "Scanning playlists: Looking for new videos.", + "message": f"Progress: {counter}/{len(all_playlists)}", + } + RedisArchivist().set_message("message:rescan", message=message) + for video in all_missing: youtube_id = video["youtube_id"] if youtube_id not in to_ignore: @@ -562,24 +552,24 @@ class VideoDownloader: def add_pending(): """add pending videos to download queue""" mess_dict = { - "status": "downloading", + "status": "message:download", "level": "info", "title": "Looking for videos to download", - "message": "", + "message": "Scanning your download queue.", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) all_pending, _ = PendingList().get_all_pending() to_add = [i["youtube_id"] for i in all_pending] if not to_add: # there is nothing pending print("download queue is empty") mess_dict = { - "status": "downloading", + "status": "message:download", "level": "error", "title": "Download queue is empty", - "message": "", + "message": "Add some videos to the queue first.", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) return queue = RedisQueue("dl_queue") @@ -606,7 +596,7 @@ class VideoDownloader: "title": title, "message": message, } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) def build_obs(self): """build obs dictionary for yt-dlp""" @@ -765,11 +755,11 @@ class VideoDownloader: ) message = f"Progress: {p_counter}/{len(all_playlist_ids)}" mess_dict = { - "status": "downloading", + "status": "message:download", "level": "info", "title": title, "message": message, } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) p_counter = p_counter + 1 c_counter = c_counter + 1 diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index 5233a60..a30a7d4 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -165,14 +165,6 @@ class Reindex: counter = 1 for channel_id in all_channel_ids: - message = f"Progress: {counter}/{len(all_channels)}" - mess_dict = { - "status": "scraping", - "level": "info", - "title": "Scraping all youtube channels", - "message": message, - } - RedisArchivist().set_message("progress:download", mess_dict) channel_index = YoutubeChannel(channel_id) subscribed = channel_index.channel_dict["channel_subscribed"] channel_index.channel_dict = channel_index.build_channel_dict( diff --git a/tubearchivist/home/src/thumbnails.py b/tubearchivist/home/src/thumbnails.py index 138cef0..a49ddc6 100644 --- a/tubearchivist/home/src/thumbnails.py +++ b/tubearchivist/home/src/thumbnails.py @@ -183,12 +183,12 @@ class ThumbManager: progress = f"{counter}/{len(missing_thumbs)}" if notify: mess_dict = { - "status": "pending", + "status": "message:download", "level": "info", - "title": "Downloading Thumbnails", - "message": "Progress: " + progress, + "title": "Processing Videos", + "message": "Downloading Thumbnails, Progress: " + progress, } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) if counter % 25 == 0: print("thumbnail progress: " + progress) @@ -213,12 +213,12 @@ class ThumbManager: img_raw.convert("RGB").save(banner_path) mess_dict = { - "status": "pending", + "status": "message:download", "level": "info", - "title": "Adding to download queue.", - "message": "Downloading Channel Art...", + "title": "Processing Channels", + "message": "Downloading Channel Art.", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) def download_playlist(self, missing_playlists): """download needed artwork for playlists""" @@ -230,12 +230,12 @@ class ThumbManager: img_raw.convert("RGB").save(thumb_path) mess_dict = { - "status": "pending", + "status": "message:download", "level": "info", - "title": "Adding to download queue.", - "message": "Downloading Playlist Art...", + "title": "Processing Playlists", + "message": "Downloading Playlist Art.", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) @staticmethod def vid_thumb_path(youtube_id): diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index f80f165..8a7c505 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -41,11 +41,12 @@ app.conf.timezone = os.environ.get("TZ") or "UTC" def update_subscribed(): """look for missing videos and add to pending""" message = { - "status": "rescan", + "status": "message:rescan", "level": "info", - "title": "Start rescanning channels and playlists.", + "title": "Rescanning channels and playlists.", + "message": "Looking for new videos." } - RedisArchivist().set_message("progress:download", message) + RedisArchivist().set_message("message:rescan", message) have_lock = False my_lock = RedisArchivist().get_lock("rescan") @@ -190,12 +191,12 @@ def kill_dl(task_id): # notify mess_dict = { - "status": "downloading", + "status": "message:download", "level": "error", - "title": "Brutally killing download queue", - "message": "", + "title": "Canceling download process", + "message": "Canceling download queue now.", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict) @shared_task diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 48ed282..b55bfac 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -303,12 +303,12 @@ class DownloadView(View): # failed to process print(f"failed to parse: {url_str}") mess_dict = { - "status": "downloading", + "status": "message:add", "level": "error", "title": "Failed to extract links.", "message": "Not a video, channel or playlist ID or URL", } - RedisArchivist().set_message("progress:download", mess_dict) + RedisArchivist().set_message("message:add", mess_dict) return redirect("downloads") print(youtube_ids)