mirror of
https://github.com/tubearchivist/tubearchivist-frontend.git
synced 2024-11-04 19:30:13 +00:00
faster release message timer with reduced redis exipre time on last message
This commit is contained in:
parent
c7d69b4fa1
commit
19871d55e2
@ -101,8 +101,8 @@ class PendingList:
|
|||||||
"""build the bulk lists"""
|
"""build the bulk lists"""
|
||||||
bulk_list = []
|
bulk_list = []
|
||||||
all_videos_added = []
|
all_videos_added = []
|
||||||
counter = 1
|
|
||||||
for youtube_id in missing_videos:
|
for idx, youtube_id in enumerate(missing_videos):
|
||||||
# check if already downloaded
|
# check if already downloaded
|
||||||
if youtube_id in self.all_downloaded:
|
if youtube_id in self.all_downloaded:
|
||||||
continue
|
continue
|
||||||
@ -114,24 +114,27 @@ class PendingList:
|
|||||||
|
|
||||||
channel_indexed = video["channel_id"] in self.all_channel_ids
|
channel_indexed = video["channel_id"] in self.all_channel_ids
|
||||||
video["channel_indexed"] = channel_indexed
|
video["channel_indexed"] = channel_indexed
|
||||||
thumb_url = video["vid_thumb_url"]
|
|
||||||
video["status"] = "pending"
|
video["status"] = "pending"
|
||||||
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
|
||||||
bulk_list.append(json.dumps(action))
|
bulk_list.append(json.dumps(action))
|
||||||
bulk_list.append(json.dumps(video))
|
bulk_list.append(json.dumps(video))
|
||||||
all_videos_added.append((youtube_id, thumb_url))
|
all_videos_added.append((youtube_id, video["vid_thumb_url"]))
|
||||||
# notify
|
# notify
|
||||||
progress = f"{counter}/{len(missing_videos)}"
|
progress = f"{idx + 1}/{len(missing_videos)}"
|
||||||
mess_dict = {
|
mess_dict = {
|
||||||
"status": "message:add",
|
"status": "message:add",
|
||||||
"level": "info",
|
"level": "info",
|
||||||
"title": "Adding new videos to download queue.",
|
"title": "Adding new videos to download queue.",
|
||||||
"message": "Progress: " + progress,
|
"message": "Progress: " + progress,
|
||||||
}
|
}
|
||||||
RedisArchivist().set_message("message:add", mess_dict)
|
if idx + 1 == len(missing_videos):
|
||||||
if counter % 25 == 0:
|
RedisArchivist().set_message(
|
||||||
|
"message:add", mess_dict, expire=4
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
RedisArchivist().set_message("message:add", mess_dict)
|
||||||
|
if idx + 1 % 25 == 0:
|
||||||
print("adding to queue progress: " + progress)
|
print("adding to queue progress: " + progress)
|
||||||
counter = counter + 1
|
|
||||||
|
|
||||||
return bulk_list, all_videos_added
|
return bulk_list, all_videos_added
|
||||||
|
|
||||||
@ -313,23 +316,28 @@ class ChannelSubscription:
|
|||||||
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
|
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
|
||||||
all_downloaded = pending_handler.get_all_downloaded()
|
all_downloaded = pending_handler.get_all_downloaded()
|
||||||
to_ignore = all_ids + all_downloaded
|
to_ignore = all_ids + all_downloaded
|
||||||
|
|
||||||
missing_videos = []
|
missing_videos = []
|
||||||
counter = 1
|
|
||||||
for channel in all_channels:
|
for idx, channel in enumerate(all_channels):
|
||||||
channel_id = channel["channel_id"]
|
channel_id = channel["channel_id"]
|
||||||
last_videos = self.get_last_youtube_videos(channel_id)
|
last_videos = self.get_last_youtube_videos(channel_id)
|
||||||
|
for video in last_videos:
|
||||||
|
if video[0] not in to_ignore:
|
||||||
|
missing_videos.append(video[0])
|
||||||
|
# notify
|
||||||
message = {
|
message = {
|
||||||
"status": "message:rescan",
|
"status": "message:rescan",
|
||||||
"level": "info",
|
"level": "info",
|
||||||
"title": "Scanning channels: Looking for new videos.",
|
"title": "Scanning channels: Looking for new videos.",
|
||||||
"message": f"Progress: {counter}/{len(all_channels)}",
|
"message": f"Progress: {idx + 1}/{len(all_channels)}",
|
||||||
}
|
}
|
||||||
RedisArchivist().set_message("message:rescan", message=message)
|
if idx + 1 == len(all_channels):
|
||||||
for video in last_videos:
|
RedisArchivist().set_message(
|
||||||
youtube_id = video[0]
|
"message:rescan", message=message, expire=4
|
||||||
if youtube_id not in to_ignore:
|
)
|
||||||
missing_videos.append(youtube_id)
|
else:
|
||||||
counter = counter + 1
|
RedisArchivist().set_message("message:rescan", message=message)
|
||||||
|
|
||||||
return missing_videos
|
return missing_videos
|
||||||
|
|
||||||
|
@ -162,8 +162,7 @@ class ThumbManager:
|
|||||||
def download_vid(self, missing_thumbs, notify=True):
|
def download_vid(self, missing_thumbs, notify=True):
|
||||||
"""download all missing thumbnails from list"""
|
"""download all missing thumbnails from list"""
|
||||||
print(f"downloading {len(missing_thumbs)} thumbnails")
|
print(f"downloading {len(missing_thumbs)} thumbnails")
|
||||||
counter = 1
|
for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
|
||||||
for youtube_id, thumb_url in missing_thumbs:
|
|
||||||
folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
|
folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
|
||||||
thumb_path = os.path.join(
|
thumb_path = os.path.join(
|
||||||
self.CACHE_DIR, self.vid_thumb_path(youtube_id)
|
self.CACHE_DIR, self.vid_thumb_path(youtube_id)
|
||||||
@ -177,10 +176,9 @@ class ThumbManager:
|
|||||||
new_height = width / 16 * 9
|
new_height = width / 16 * 9
|
||||||
offset = (height - new_height) / 2
|
offset = (height - new_height) / 2
|
||||||
img_raw = img_raw.crop((0, offset, width, height - offset))
|
img_raw = img_raw.crop((0, offset, width, height - offset))
|
||||||
|
|
||||||
img_raw.convert("RGB").save(thumb_path)
|
img_raw.convert("RGB").save(thumb_path)
|
||||||
|
|
||||||
progress = f"{counter}/{len(missing_thumbs)}"
|
progress = f"{idx + 1}/{len(missing_thumbs)}"
|
||||||
if notify:
|
if notify:
|
||||||
mess_dict = {
|
mess_dict = {
|
||||||
"status": "message:add",
|
"status": "message:add",
|
||||||
@ -188,11 +186,15 @@ class ThumbManager:
|
|||||||
"title": "Processing Videos",
|
"title": "Processing Videos",
|
||||||
"message": "Downloading Thumbnails, Progress: " + progress,
|
"message": "Downloading Thumbnails, Progress: " + progress,
|
||||||
}
|
}
|
||||||
RedisArchivist().set_message("message:add", mess_dict)
|
if idx + 1 == len(missing_thumbs):
|
||||||
|
RedisArchivist().set_message(
|
||||||
|
"message:add", mess_dict, expire=4
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
RedisArchivist().set_message("message:add", mess_dict)
|
||||||
|
|
||||||
if counter % 25 == 0:
|
if idx + 1 % 25 == 0:
|
||||||
print("thumbnail progress: " + progress)
|
print("thumbnail progress: " + progress)
|
||||||
counter = counter + 1
|
|
||||||
|
|
||||||
def download_chan(self, missing_channels):
|
def download_chan(self, missing_channels):
|
||||||
"""download needed artwork for channels"""
|
"""download needed artwork for channels"""
|
||||||
|
Loading…
Reference in New Issue
Block a user