faster release message timer with reduced redis exipre time on last message

This commit is contained in:
simon 2021-12-05 21:27:43 +07:00
parent c7d69b4fa1
commit 19871d55e2
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 34 additions and 24 deletions

View File

@ -101,8 +101,8 @@ class PendingList:
"""build the bulk lists""" """build the bulk lists"""
bulk_list = [] bulk_list = []
all_videos_added = [] all_videos_added = []
counter = 1
for youtube_id in missing_videos: for idx, youtube_id in enumerate(missing_videos):
# check if already downloaded # check if already downloaded
if youtube_id in self.all_downloaded: if youtube_id in self.all_downloaded:
continue continue
@ -114,24 +114,27 @@ class PendingList:
channel_indexed = video["channel_id"] in self.all_channel_ids channel_indexed = video["channel_id"] in self.all_channel_ids
video["channel_indexed"] = channel_indexed video["channel_indexed"] = channel_indexed
thumb_url = video["vid_thumb_url"]
video["status"] = "pending" video["status"] = "pending"
action = {"create": {"_id": youtube_id, "_index": "ta_download"}} action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
bulk_list.append(json.dumps(action)) bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(video)) bulk_list.append(json.dumps(video))
all_videos_added.append((youtube_id, thumb_url)) all_videos_added.append((youtube_id, video["vid_thumb_url"]))
# notify # notify
progress = f"{counter}/{len(missing_videos)}" progress = f"{idx + 1}/{len(missing_videos)}"
mess_dict = { mess_dict = {
"status": "message:add", "status": "message:add",
"level": "info", "level": "info",
"title": "Adding new videos to download queue.", "title": "Adding new videos to download queue.",
"message": "Progress: " + progress, "message": "Progress: " + progress,
} }
RedisArchivist().set_message("message:add", mess_dict) if idx + 1 == len(missing_videos):
if counter % 25 == 0: RedisArchivist().set_message(
"message:add", mess_dict, expire=4
)
else:
RedisArchivist().set_message("message:add", mess_dict)
if idx + 1 % 25 == 0:
print("adding to queue progress: " + progress) print("adding to queue progress: " + progress)
counter = counter + 1
return bulk_list, all_videos_added return bulk_list, all_videos_added
@ -313,23 +316,28 @@ class ChannelSubscription:
all_ids = [i["youtube_id"] for i in all_ignore + all_pending] all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
all_downloaded = pending_handler.get_all_downloaded() all_downloaded = pending_handler.get_all_downloaded()
to_ignore = all_ids + all_downloaded to_ignore = all_ids + all_downloaded
missing_videos = [] missing_videos = []
counter = 1
for channel in all_channels: for idx, channel in enumerate(all_channels):
channel_id = channel["channel_id"] channel_id = channel["channel_id"]
last_videos = self.get_last_youtube_videos(channel_id) last_videos = self.get_last_youtube_videos(channel_id)
for video in last_videos:
if video[0] not in to_ignore:
missing_videos.append(video[0])
# notify
message = { message = {
"status": "message:rescan", "status": "message:rescan",
"level": "info", "level": "info",
"title": "Scanning channels: Looking for new videos.", "title": "Scanning channels: Looking for new videos.",
"message": f"Progress: {counter}/{len(all_channels)}", "message": f"Progress: {idx + 1}/{len(all_channels)}",
} }
RedisArchivist().set_message("message:rescan", message=message) if idx + 1 == len(all_channels):
for video in last_videos: RedisArchivist().set_message(
youtube_id = video[0] "message:rescan", message=message, expire=4
if youtube_id not in to_ignore: )
missing_videos.append(youtube_id) else:
counter = counter + 1 RedisArchivist().set_message("message:rescan", message=message)
return missing_videos return missing_videos

View File

@ -162,8 +162,7 @@ class ThumbManager:
def download_vid(self, missing_thumbs, notify=True): def download_vid(self, missing_thumbs, notify=True):
"""download all missing thumbnails from list""" """download all missing thumbnails from list"""
print(f"downloading {len(missing_thumbs)} thumbnails") print(f"downloading {len(missing_thumbs)} thumbnails")
counter = 1 for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
for youtube_id, thumb_url in missing_thumbs:
folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower()) folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
thumb_path = os.path.join( thumb_path = os.path.join(
self.CACHE_DIR, self.vid_thumb_path(youtube_id) self.CACHE_DIR, self.vid_thumb_path(youtube_id)
@ -177,10 +176,9 @@ class ThumbManager:
new_height = width / 16 * 9 new_height = width / 16 * 9
offset = (height - new_height) / 2 offset = (height - new_height) / 2
img_raw = img_raw.crop((0, offset, width, height - offset)) img_raw = img_raw.crop((0, offset, width, height - offset))
img_raw.convert("RGB").save(thumb_path) img_raw.convert("RGB").save(thumb_path)
progress = f"{counter}/{len(missing_thumbs)}" progress = f"{idx + 1}/{len(missing_thumbs)}"
if notify: if notify:
mess_dict = { mess_dict = {
"status": "message:add", "status": "message:add",
@ -188,11 +186,15 @@ class ThumbManager:
"title": "Processing Videos", "title": "Processing Videos",
"message": "Downloading Thumbnails, Progress: " + progress, "message": "Downloading Thumbnails, Progress: " + progress,
} }
RedisArchivist().set_message("message:add", mess_dict) if idx + 1 == len(missing_thumbs):
RedisArchivist().set_message(
"message:add", mess_dict, expire=4
)
else:
RedisArchivist().set_message("message:add", mess_dict)
if counter % 25 == 0: if idx + 1 % 25 == 0:
print("thumbnail progress: " + progress) print("thumbnail progress: " + progress)
counter = counter + 1
def download_chan(self, missing_channels): def download_chan(self, missing_channels):
"""download needed artwork for channels""" """download needed artwork for channels"""