tubearchivist/tubearchivist-frontend
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist-frontend.git synced 2024-11-22 20:00:15 +00:00
Code Issues Projects Releases Wiki Activity

transform add to queue, rescan, and download progress to channel messages

Browse Source
This commit is contained in:
simon 2021-12-03 19:47:43 +07:00
parent 1691bdadf5
commit b35440253d
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
5 changed files with 50 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tubearchivist/home/src/download.py
View File

@ -48,12 +48,12 @@ class PendingList:
for entry in youtube_ids: for entry in youtube_ids:
# notify # notify
mess_dict = { mess_dict = {
"status": "pending", "status": "message:add",
"level": "info", "level": "info",
"title": "Adding to download queue.", "title": "Adding to download queue.",
"message": "Extracting lists", "message": "Extracting lists",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:add", mess_dict)
# extract # extract
url = entry["url"] url = entry["url"]
url_type = entry["type"] url_type = entry["type"]
@ -123,12 +123,12 @@ class PendingList:
# notify # notify
progress = f"{counter}/{len(missing_videos)}" progress = f"{counter}/{len(missing_videos)}"
mess_dict = { mess_dict = {
"status": "pending", "status": "message:add",
"level": "info", "level": "info",
"title": "Adding new videos to download queue.", "title": "Adding new videos to download queue.",
"message": "Progress: " + progress, "message": "Progress: " + progress,
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:add", mess_dict)
if counter % 25 == 0: if counter % 25 == 0:
print("adding to queue progress: " + progress) print("adding to queue progress: " + progress)
counter = counter + 1 counter = counter + 1
@ -261,13 +261,6 @@ class PendingList:
request = requests.post( request = requests.post(
url, data=query_str, headers=headers, auth=self.ES_AUTH url, data=query_str, headers=headers, auth=self.ES_AUTH
) )
mess_dict = {
"status": "ignore",
"level": "info",
"title": "Added to ignore list",
"message": "",
}
RedisArchivist().set_message("progress:download", mess_dict)
if not request.ok: if not request.ok:
print(request) print(request)
raise ValueError("failed to set video to ignore") raise ValueError("failed to set video to ignore")
@ -325,15 +318,13 @@ class ChannelSubscription:
for channel in all_channels: for channel in all_channels:
channel_id = channel["channel_id"] channel_id = channel["channel_id"]
last_videos = self.get_last_youtube_videos(channel_id) last_videos = self.get_last_youtube_videos(channel_id)
RedisArchivist().set_message( message = {
"progress:download", "status": "message:rescan",
{
"status": "rescan",
"level": "info", "level": "info",
"title": "Scanning channels: Looking for new videos.", "title": "Scanning channels: Looking for new videos.",
"message": f"Progress: {counter}/{len(all_channels)}", "message": f"Progress: {counter}/{len(all_channels)}",
}, }
) RedisArchivist().set_message("message:rescan", message=message)
for video in last_videos: for video in last_videos:
youtube_id = video[0] youtube_id = video[0]
if youtube_id not in to_ignore: if youtube_id not in to_ignore:
@ -506,15 +497,14 @@ class PlaylistSubscription:
playlist_entries = playlist["playlist_entries"] playlist_entries = playlist["playlist_entries"]
all_missing = [i for i in playlist_entries if not i["downloaded"]] all_missing = [i for i in playlist_entries if not i["downloaded"]]
RedisArchivist().set_message( message = {
"progress:download", "status": "message:rescan",
{
"status": "rescan",
"level": "info", "level": "info",
"title": "Scanning playlists: Looking for new videos.", "title": "Scanning playlists: Looking for new videos.",
"message": f"Progress: {counter}/{len(all_playlists)}", "message": f"Progress: {counter}/{len(all_playlists)}",
}, }
) RedisArchivist().set_message("message:rescan", message=message)
for video in all_missing: for video in all_missing:
youtube_id = video["youtube_id"] youtube_id = video["youtube_id"]
if youtube_id not in to_ignore: if youtube_id not in to_ignore:
@ -562,24 +552,24 @@ class VideoDownloader:
def add_pending(): def add_pending():
"""add pending videos to download queue""" """add pending videos to download queue"""
mess_dict = { mess_dict = {
"status": "downloading", "status": "message:download",
"level": "info", "level": "info",
"title": "Looking for videos to download", "title": "Looking for videos to download",
"message": "", "message": "Scanning your download queue.",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
all_pending, _ = PendingList().get_all_pending() all_pending, _ = PendingList().get_all_pending()
to_add = [i["youtube_id"] for i in all_pending] to_add = [i["youtube_id"] for i in all_pending]
if not to_add: if not to_add:
# there is nothing pending # there is nothing pending
print("download queue is empty") print("download queue is empty")
mess_dict = { mess_dict = {
"status": "downloading", "status": "message:download",
"level": "error", "level": "error",
"title": "Download queue is empty", "title": "Download queue is empty",
"message": "", "message": "Add some videos to the queue first.",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
return return
queue = RedisQueue("dl_queue") queue = RedisQueue("dl_queue")
@ -606,7 +596,7 @@ class VideoDownloader:
"title": title, "title": title,
"message": message, "message": message,
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
def build_obs(self): def build_obs(self):
"""build obs dictionary for yt-dlp""" """build obs dictionary for yt-dlp"""
@ -765,11 +755,11 @@ class VideoDownloader:
) )
message = f"Progress: {p_counter}/{len(all_playlist_ids)}" message = f"Progress: {p_counter}/{len(all_playlist_ids)}"
mess_dict = { mess_dict = {
"status": "downloading", "status": "message:download",
"level": "info", "level": "info",
"title": title, "title": title,
"message": message, "message": message,
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
p_counter = p_counter + 1 p_counter = p_counter + 1
c_counter = c_counter + 1 c_counter = c_counter + 1

8
tubearchivist/home/src/reindex.py
View File

@ -165,14 +165,6 @@ class Reindex:
counter = 1 counter = 1
for channel_id in all_channel_ids: for channel_id in all_channel_ids:
message = f"Progress: {counter}/{len(all_channels)}"
mess_dict = {
"status": "scraping",
"level": "info",
"title": "Scraping all youtube channels",
"message": message,
}
RedisArchivist().set_message("progress:download", mess_dict)
channel_index = YoutubeChannel(channel_id) channel_index = YoutubeChannel(channel_id)
subscribed = channel_index.channel_dict["channel_subscribed"] subscribed = channel_index.channel_dict["channel_subscribed"]
channel_index.channel_dict = channel_index.build_channel_dict( channel_index.channel_dict = channel_index.build_channel_dict(

24
tubearchivist/home/src/thumbnails.py
View File

@ -183,12 +183,12 @@ class ThumbManager:
progress = f"{counter}/{len(missing_thumbs)}" progress = f"{counter}/{len(missing_thumbs)}"
if notify: if notify:
mess_dict = { mess_dict = {
"status": "pending", "status": "message:download",
"level": "info", "level": "info",
"title": "Downloading Thumbnails", "title": "Processing Videos",
"message": "Progress: " + progress, "message": "Downloading Thumbnails, Progress: " + progress,
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
if counter % 25 == 0: if counter % 25 == 0:
print("thumbnail progress: " + progress) print("thumbnail progress: " + progress)
@ -213,12 +213,12 @@ class ThumbManager:
img_raw.convert("RGB").save(banner_path) img_raw.convert("RGB").save(banner_path)
mess_dict = { mess_dict = {
"status": "pending", "status": "message:download",
"level": "info", "level": "info",
"title": "Adding to download queue.", "title": "Processing Channels",
"message": "Downloading Channel Art...", "message": "Downloading Channel Art.",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
def download_playlist(self, missing_playlists): def download_playlist(self, missing_playlists):
"""download needed artwork for playlists""" """download needed artwork for playlists"""
@ -230,12 +230,12 @@ class ThumbManager:
img_raw.convert("RGB").save(thumb_path) img_raw.convert("RGB").save(thumb_path)
mess_dict = { mess_dict = {
"status": "pending", "status": "message:download",
"level": "info", "level": "info",
"title": "Adding to download queue.", "title": "Processing Playlists",
"message": "Downloading Playlist Art...", "message": "Downloading Playlist Art.",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
@staticmethod @staticmethod
def vid_thumb_path(youtube_id): def vid_thumb_path(youtube_id):

15
tubearchivist/home/tasks.py
View File

@ -41,11 +41,12 @@ app.conf.timezone = os.environ.get("TZ") or "UTC"
def update_subscribed(): def update_subscribed():
"""look for missing videos and add to pending""" """look for missing videos and add to pending"""
message = { message = {
"status": "rescan", "status": "message:rescan",
"level": "info", "level": "info",
"title": "Start rescanning channels and playlists.", "title": "Rescanning channels and playlists.",
"message": "Looking for new videos."
} }
RedisArchivist().set_message("progress:download", message) RedisArchivist().set_message("message:rescan", message)
have_lock = False have_lock = False
my_lock = RedisArchivist().get_lock("rescan") my_lock = RedisArchivist().get_lock("rescan")
@ -190,12 +191,12 @@ def kill_dl(task_id):
# notify # notify
mess_dict = { mess_dict = {
"status": "downloading", "status": "message:download",
"level": "error", "level": "error",
"title": "Brutally killing download queue", "title": "Canceling download process",
"message": "", "message": "Canceling download queue now.",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
@shared_task @shared_task

4
tubearchivist/home/views.py
View File

@ -303,12 +303,12 @@ class DownloadView(View):
# failed to process # failed to process
print(f"failed to parse: {url_str}") print(f"failed to parse: {url_str}")
mess_dict = { mess_dict = {
"status": "downloading", "status": "message:add",
"level": "error", "level": "error",
"title": "Failed to extract links.", "title": "Failed to extract links.",
"message": "Not a video, channel or playlist ID or URL", "message": "Not a video, channel or playlist ID or URL",
} }
RedisArchivist().set_message("progress:download", mess_dict) RedisArchivist().set_message("message:add", mess_dict)
return redirect("downloads") return redirect("downloads")
print(youtube_ids) print(youtube_ids)