faster release message timer with reduced redis exipre time on last message

tubearchivist/home/src/download.py

@@ -101,8 +101,8 @@ class PendingList:
         """build the bulk lists"""
         bulk_list = []
         all_videos_added = []
-        counter = 1
-        for youtube_id in missing_videos:
+
+        for idx, youtube_id in enumerate(missing_videos):
             # check if already downloaded
             if youtube_id in self.all_downloaded:
                 continue
@@ -114,24 +114,27 @@ class PendingList:
 
             channel_indexed = video["channel_id"] in self.all_channel_ids
             video["channel_indexed"] = channel_indexed
-            thumb_url = video["vid_thumb_url"]
             video["status"] = "pending"
             action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
             bulk_list.append(json.dumps(action))
             bulk_list.append(json.dumps(video))
-            all_videos_added.append((youtube_id, thumb_url))
+            all_videos_added.append((youtube_id, video["vid_thumb_url"]))
             # notify
-            progress = f"{counter}/{len(missing_videos)}"
+            progress = f"{idx + 1}/{len(missing_videos)}"
             mess_dict = {
                 "status": "message:add",
                 "level": "info",
                 "title": "Adding new videos to download queue.",
                 "message": "Progress: " + progress,
             }
-            RedisArchivist().set_message("message:add", mess_dict)
-            if counter % 25 == 0:
+            if idx + 1 == len(missing_videos):
+                RedisArchivist().set_message(
+                    "message:add", mess_dict, expire=4
+                )
+            else:
+                RedisArchivist().set_message("message:add", mess_dict)
+            if idx + 1 % 25 == 0:
                 print("adding to queue progress: " + progress)
-            counter = counter + 1
 
         return bulk_list, all_videos_added
 
@@ -313,23 +316,28 @@ class ChannelSubscription:
         all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
         all_downloaded = pending_handler.get_all_downloaded()
         to_ignore = all_ids + all_downloaded
+
         missing_videos = []
-        counter = 1
-        for channel in all_channels:
+
+        for idx, channel in enumerate(all_channels):
             channel_id = channel["channel_id"]
             last_videos = self.get_last_youtube_videos(channel_id)
+            for video in last_videos:
+                if video[0] not in to_ignore:
+                    missing_videos.append(video[0])
+            # notify
             message = {
                 "status": "message:rescan",
                 "level": "info",
                 "title": "Scanning channels: Looking for new videos.",
-                "message": f"Progress: {counter}/{len(all_channels)}",
+                "message": f"Progress: {idx + 1}/{len(all_channels)}",
             }
-            RedisArchivist().set_message("message:rescan", message=message)
-            for video in last_videos:
-                youtube_id = video[0]
-                if youtube_id not in to_ignore:
-                    missing_videos.append(youtube_id)
-            counter = counter + 1
+            if idx + 1 == len(all_channels):
+                RedisArchivist().set_message(
+                    "message:rescan", message=message, expire=4
+                )
+            else:
+                RedisArchivist().set_message("message:rescan", message=message)
 
         return missing_videos
 

tubearchivist/home/src/thumbnails.py

@@ -162,8 +162,7 @@ class ThumbManager:
     def download_vid(self, missing_thumbs, notify=True):
         """download all missing thumbnails from list"""
         print(f"downloading {len(missing_thumbs)} thumbnails")
-        counter = 1
-        for youtube_id, thumb_url in missing_thumbs:
+        for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
             folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
             thumb_path = os.path.join(
                 self.CACHE_DIR, self.vid_thumb_path(youtube_id)
@@ -177,10 +176,9 @@ class ThumbManager:
                 new_height = width / 16 * 9
                 offset = (height - new_height) / 2
                 img_raw = img_raw.crop((0, offset, width, height - offset))
-
             img_raw.convert("RGB").save(thumb_path)
 
-            progress = f"{counter}/{len(missing_thumbs)}"
+            progress = f"{idx + 1}/{len(missing_thumbs)}"
             if notify:
                 mess_dict = {
                     "status": "message:add",
@@ -188,11 +186,15 @@ class ThumbManager:
                     "title": "Processing Videos",
                     "message": "Downloading Thumbnails, Progress: " + progress,
                 }
-                RedisArchivist().set_message("message:add", mess_dict)
+                if idx + 1 == len(missing_thumbs):
+                    RedisArchivist().set_message(
+                        "message:add", mess_dict, expire=4
+                    )
+                else:
+                    RedisArchivist().set_message("message:add", mess_dict)
 
-            if counter % 25 == 0:
+            if idx + 1 % 25 == 0:
                 print("thumbnail progress: " + progress)
-            counter = counter + 1
 
     def download_chan(self, missing_channels):
         """download needed artwork for channels"""