tubearchivist/tubearchivist
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist.git synced 2024-12-21 09:20:12 +00:00
Code Issues Projects Releases Wiki Activity

faster release message timer with reduced redis exipre time on last message

Browse Source
This commit is contained in:
simon 2021-12-05 21:27:43 +07:00
parent c7d69b4fa1
commit 19871d55e2
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 34 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
tubearchivist/home/src/download.py
View File

@ -101,8 +101,8 @@ class PendingList:
"""build the bulk lists"""
bulk_list = []
all_videos_added = []
counter = 1
for youtube_id in missing_videos:
for idx, youtube_id in enumerate(missing_videos):
# check if already downloaded
if youtube_id in self.all_downloaded:
continue
@ -114,24 +114,27 @@ class PendingList:
channel_indexed = video["channel_id"] in self.all_channel_ids
video["channel_indexed"] = channel_indexed
thumb_url = video["vid_thumb_url"]
video["status"] = "pending"
action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(video))
all_videos_added.append((youtube_id, thumb_url))
all_videos_added.append((youtube_id, video["vid_thumb_url"]))
# notify
progress = f"{counter}/{len(missing_videos)}"
progress = f"{idx + 1}/{len(missing_videos)}"
mess_dict = {
"status": "message:add",
"level": "info",
"title": "Adding new videos to download queue.",
"message": "Progress: " + progress,
}
RedisArchivist().set_message("message:add", mess_dict)
if counter % 25 == 0:
if idx + 1 == len(missing_videos):
RedisArchivist().set_message(
"message:add", mess_dict, expire=4
)
else:
RedisArchivist().set_message("message:add", mess_dict)
if idx + 1 % 25 == 0:
print("adding to queue progress: " + progress)
counter = counter + 1
return bulk_list, all_videos_added
@ -313,23 +316,28 @@ class ChannelSubscription:
all_ids = [i["youtube_id"] for i in all_ignore + all_pending]
all_downloaded = pending_handler.get_all_downloaded()
to_ignore = all_ids + all_downloaded
missing_videos = []
counter = 1
for channel in all_channels:
for idx, channel in enumerate(all_channels):
channel_id = channel["channel_id"]
last_videos = self.get_last_youtube_videos(channel_id)
for video in last_videos:
if video[0] not in to_ignore:
missing_videos.append(video[0])
# notify
message = {
"status": "message:rescan",
"level": "info",
"title": "Scanning channels: Looking for new videos.",
"message": f"Progress: {counter}/{len(all_channels)}",
"message": f"Progress: {idx + 1}/{len(all_channels)}",
}
RedisArchivist().set_message("message:rescan", message=message)
for video in last_videos:
youtube_id = video[0]
if youtube_id not in to_ignore:
missing_videos.append(youtube_id)
counter = counter + 1
if idx + 1 == len(all_channels):
RedisArchivist().set_message(
"message:rescan", message=message, expire=4
)
else:
RedisArchivist().set_message("message:rescan", message=message)
return missing_videos

16
tubearchivist/home/src/thumbnails.py
View File

@ -162,8 +162,7 @@ class ThumbManager:
def download_vid(self, missing_thumbs, notify=True):
"""download all missing thumbnails from list"""
print(f"downloading {len(missing_thumbs)} thumbnails")
counter = 1
for youtube_id, thumb_url in missing_thumbs:
for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
thumb_path = os.path.join(
self.CACHE_DIR, self.vid_thumb_path(youtube_id)
@ -177,10 +176,9 @@ class ThumbManager:
new_height = width / 16 * 9
offset = (height - new_height) / 2
img_raw = img_raw.crop((0, offset, width, height - offset))
img_raw.convert("RGB").save(thumb_path)
progress = f"{counter}/{len(missing_thumbs)}"
progress = f"{idx + 1}/{len(missing_thumbs)}"
if notify:
mess_dict = {
"status": "message:add",
@ -188,11 +186,15 @@ class ThumbManager:
"title": "Processing Videos",
"message": "Downloading Thumbnails, Progress: " + progress,
}
RedisArchivist().set_message("message:add", mess_dict)
if idx + 1 == len(missing_thumbs):
RedisArchivist().set_message(
"message:add", mess_dict, expire=4
)
else:
RedisArchivist().set_message("message:add", mess_dict)
if counter % 25 == 0:
if idx + 1 % 25 == 0:
print("thumbnail progress: " + progress)
counter = counter + 1
def download_chan(self, missing_channels):
"""download needed artwork for channels"""