From 19871d55e2dfcc4b4e5fe9ce7bd2e1518043197f Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 5 Dec 2021 21:27:43 +0700 Subject: [PATCH] faster release message timer with reduced redis exipre time on last message --- tubearchivist/home/src/download.py | 42 +++++++++++++++++----------- tubearchivist/home/src/thumbnails.py | 16 ++++++----- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/tubearchivist/home/src/download.py b/tubearchivist/home/src/download.py index 7e5fc6c..cb546ec 100644 --- a/tubearchivist/home/src/download.py +++ b/tubearchivist/home/src/download.py @@ -101,8 +101,8 @@ class PendingList: """build the bulk lists""" bulk_list = [] all_videos_added = [] - counter = 1 - for youtube_id in missing_videos: + + for idx, youtube_id in enumerate(missing_videos): # check if already downloaded if youtube_id in self.all_downloaded: continue @@ -114,24 +114,27 @@ class PendingList: channel_indexed = video["channel_id"] in self.all_channel_ids video["channel_indexed"] = channel_indexed - thumb_url = video["vid_thumb_url"] video["status"] = "pending" action = {"create": {"_id": youtube_id, "_index": "ta_download"}} bulk_list.append(json.dumps(action)) bulk_list.append(json.dumps(video)) - all_videos_added.append((youtube_id, thumb_url)) + all_videos_added.append((youtube_id, video["vid_thumb_url"])) # notify - progress = f"{counter}/{len(missing_videos)}" + progress = f"{idx + 1}/{len(missing_videos)}" mess_dict = { "status": "message:add", "level": "info", "title": "Adding new videos to download queue.", "message": "Progress: " + progress, } - RedisArchivist().set_message("message:add", mess_dict) - if counter % 25 == 0: + if idx + 1 == len(missing_videos): + RedisArchivist().set_message( + "message:add", mess_dict, expire=4 + ) + else: + RedisArchivist().set_message("message:add", mess_dict) + if idx + 1 % 25 == 0: print("adding to queue progress: " + progress) - counter = counter + 1 return bulk_list, all_videos_added @@ -313,23 +316,28 @@ class ChannelSubscription: all_ids = [i["youtube_id"] for i in all_ignore + all_pending] all_downloaded = pending_handler.get_all_downloaded() to_ignore = all_ids + all_downloaded + missing_videos = [] - counter = 1 - for channel in all_channels: + + for idx, channel in enumerate(all_channels): channel_id = channel["channel_id"] last_videos = self.get_last_youtube_videos(channel_id) + for video in last_videos: + if video[0] not in to_ignore: + missing_videos.append(video[0]) + # notify message = { "status": "message:rescan", "level": "info", "title": "Scanning channels: Looking for new videos.", - "message": f"Progress: {counter}/{len(all_channels)}", + "message": f"Progress: {idx + 1}/{len(all_channels)}", } - RedisArchivist().set_message("message:rescan", message=message) - for video in last_videos: - youtube_id = video[0] - if youtube_id not in to_ignore: - missing_videos.append(youtube_id) - counter = counter + 1 + if idx + 1 == len(all_channels): + RedisArchivist().set_message( + "message:rescan", message=message, expire=4 + ) + else: + RedisArchivist().set_message("message:rescan", message=message) return missing_videos diff --git a/tubearchivist/home/src/thumbnails.py b/tubearchivist/home/src/thumbnails.py index c8d998e..d9f6549 100644 --- a/tubearchivist/home/src/thumbnails.py +++ b/tubearchivist/home/src/thumbnails.py @@ -162,8 +162,7 @@ class ThumbManager: def download_vid(self, missing_thumbs, notify=True): """download all missing thumbnails from list""" print(f"downloading {len(missing_thumbs)} thumbnails") - counter = 1 - for youtube_id, thumb_url in missing_thumbs: + for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs): folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower()) thumb_path = os.path.join( self.CACHE_DIR, self.vid_thumb_path(youtube_id) @@ -177,10 +176,9 @@ class ThumbManager: new_height = width / 16 * 9 offset = (height - new_height) / 2 img_raw = img_raw.crop((0, offset, width, height - offset)) - img_raw.convert("RGB").save(thumb_path) - progress = f"{counter}/{len(missing_thumbs)}" + progress = f"{idx + 1}/{len(missing_thumbs)}" if notify: mess_dict = { "status": "message:add", @@ -188,11 +186,15 @@ class ThumbManager: "title": "Processing Videos", "message": "Downloading Thumbnails, Progress: " + progress, } - RedisArchivist().set_message("message:add", mess_dict) + if idx + 1 == len(missing_thumbs): + RedisArchivist().set_message( + "message:add", mess_dict, expire=4 + ) + else: + RedisArchivist().set_message("message:add", mess_dict) - if counter % 25 == 0: + if idx + 1 % 25 == 0: print("thumbnail progress: " + progress) - counter = counter + 1 def download_chan(self, missing_channels): """download needed artwork for channels"""