From 2cf30e1127d5e15f35da8d147d7a63815d845329 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 16 Jun 2022 10:37:46 +0700 Subject: [PATCH] refactor: default set_message in RedisArchivist to True --- tubearchivist/api/src/task_processor.py | 2 +- tubearchivist/api/views.py | 4 +- tubearchivist/home/src/download/queue.py | 7 ++-- .../home/src/download/subscriptions.py | 16 +++++--- tubearchivist/home/src/download/thumbnails.py | 16 +++++--- .../home/src/download/yt_dlp_base.py | 6 +-- .../home/src/download/yt_dlp_handler.py | 41 +++++++++---------- tubearchivist/home/src/frontend/api_calls.py | 18 ++++---- tubearchivist/home/src/index/channel.py | 9 ++-- tubearchivist/home/src/index/filesystem.py | 2 +- tubearchivist/home/src/index/video.py | 2 +- tubearchivist/home/src/ta/config.py | 19 +++++---- tubearchivist/home/src/ta/ta_redis.py | 2 +- tubearchivist/home/tasks.py | 19 +++++---- tubearchivist/home/views.py | 24 +++++------ 15 files changed, 101 insertions(+), 86 deletions(-) diff --git a/tubearchivist/api/src/task_processor.py b/tubearchivist/api/src/task_processor.py index f13b953..dd42ee0 100644 --- a/tubearchivist/api/src/task_processor.py +++ b/tubearchivist/api/src/task_processor.py @@ -50,5 +50,5 @@ class TaskHandler: print("download pending") running = download_pending.delay() print("set task id: " + running.id) - RedisArchivist().set_message("dl_queue_id", running.id, expire=False) + RedisArchivist().set_message("dl_queue_id", running.id) return {"success": True} diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index f695fab..be9c571 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -131,7 +131,7 @@ class VideoProgressView(ApiBaseView): position = request.data.get("position", 0) key = f"{request.user.id}:progress:{video_id}" message = {"position": position, "youtube_id": video_id} - RedisArchivist().set_message(key, message, expire=False) + RedisArchivist().set_message(key, message) self.response = request.data return Response(self.response) @@ -459,7 +459,7 @@ class TaskApiView(ApiBaseView): @staticmethod def get(request): """handle get request""" - + # pylint: disable=unused-argument response = {"rescan": False, "downloading": False} for key in response.keys(): response[key] = RedisArchivist().is_locked(key) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index c115fbc..b318b12 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -150,7 +150,7 @@ class PendingList(PendingIndex): "title": "Adding to download queue.", "message": "Extracting lists", } - RedisArchivist().set_message("message:add", mess_dict) + RedisArchivist().set_message("message:add", mess_dict, expire=True) self._process_entry(entry) def _process_entry(self, entry): @@ -229,10 +229,11 @@ class PendingList(PendingIndex): "message": "Progress: " + progress, } if idx + 1 == len(self.missing_videos): - RedisArchivist().set_message("message:add", mess_dict, expire=4) + expire = 4 else: - RedisArchivist().set_message("message:add", mess_dict) + expire = True + RedisArchivist().set_message("message:add", mess_dict, expire=expire) if idx + 1 % 25 == 0: print("adding to queue progress: " + progress) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 916c170..d353ed0 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -76,11 +76,13 @@ class ChannelSubscription: "message": f"Progress: {idx + 1}/{len(all_channels)}", } if idx + 1 == len(all_channels): - RedisArchivist().set_message( - "message:rescan", message=message, expire=4 - ) + expire = 4 else: - RedisArchivist().set_message("message:rescan", message=message) + expire = True + + RedisArchivist().set_message( + "message:rescan", message=message, expire=expire + ) return missing_videos @@ -152,7 +154,7 @@ class PlaylistSubscription: "message": f"Processing {idx + 1} of {len(new_playlists)}", } RedisArchivist().set_message( - "message:subplaylist", message=message + "message:subplaylist", message=message, expire=True ) return new_thumbs @@ -206,7 +208,9 @@ class PlaylistSubscription: "title": "Scanning playlists: Looking for new videos.", "message": f"Progress: {idx + 1}/{len(all_playlists)}", } - RedisArchivist().set_message("message:rescan", message=message) + RedisArchivist().set_message( + "message:rescan", message=message, expire=True + ) for video in all_missing: youtube_id = video["youtube_id"] diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 5df6000..c317ee4 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -193,11 +193,13 @@ class ThumbManager: "message": "Downloading Thumbnails, Progress: " + progress, } if idx + 1 == len(missing_thumbs): - RedisArchivist().set_message( - "message:add", mess_dict, expire=4 - ) + expire = 4 else: - RedisArchivist().set_message("message:add", mess_dict) + expire = True + + RedisArchivist().set_message( + "message:add", mess_dict, expire=expire + ) if idx + 1 % 25 == 0: print("thumbnail progress: " + progress) @@ -226,7 +228,8 @@ class ThumbManager: "title": "Processing Channels", "message": "Downloading Channel Art.", } - RedisArchivist().set_message("message:download", mess_dict) + key = "message:download" + RedisArchivist().set_message(key, mess_dict, expire=True) def download_playlist(self, missing_playlists): """download needed artwork for playlists""" @@ -243,7 +246,8 @@ class ThumbManager: "title": "Processing Playlists", "message": "Downloading Playlist Art.", } - RedisArchivist().set_message("message:download", mess_dict) + key = "message:download" + RedisArchivist().set_message(key, mess_dict, expire=True) def get_base64_blur(self, youtube_id): """return base64 encoded placeholder""" diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 54f61de..b8d9aa3 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -85,7 +85,7 @@ class CookieHandler: with open(import_path, encoding="utf-8") as cookie_file: cookie = cookie_file.read() - RedisArchivist().set_message("cookie", cookie, expire=False) + RedisArchivist().set_message("cookie", cookie) os.remove(import_path) print("cookie: import successful") @@ -95,7 +95,7 @@ class CookieHandler: """revoke cookie""" RedisArchivist().del_message("cookie") RedisArchivist().set_message( - "config", False, path=".downloads.cookie_import", expire=False + "config", False, path=".downloads.cookie_import" ) print("cookie: revoked") @@ -112,7 +112,7 @@ class CookieHandler: # update in redis to avoid expiring modified = validator.obs["cookiefile"].getvalue() if modified: - RedisArchivist().set_message("cookie", modified, expire=False) + RedisArchivist().set_message("cookie", modified) if not response: mess_dict = { diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 482463c..49db964 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -125,18 +125,19 @@ class DownloadPostProcess: + f"{id_c + 1}/{len(self.download.channels)}" ) message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}" + key = "message:download" mess_dict = { - "status": "message:download", + "status": key, "level": "info", "title": title, "message": message, } if id_p + 1 == len(all_channel_playlist): - RedisArchivist().set_message( - "message:download", mess_dict, expire=4 - ) + expire = 4 else: - RedisArchivist().set_message("message:download", mess_dict) + expire = True + + RedisArchivist().set_message(key, mess_dict, expire=expire) class VideoDownloader: @@ -145,6 +146,8 @@ class VideoDownloader: if not initiated with list, take from queue """ + MSG = "message:download" + def __init__(self, youtube_id_list=False): self.obs = False self.video_overwrites = False @@ -177,23 +180,21 @@ class VideoDownloader: ) self.channels.add(vid_dict["channel"]["channel_id"]) mess_dict = { - "status": "message:download", + "status": self.MSG, "level": "info", "title": "Moving....", "message": "Moving downloaded file to storage folder", } - RedisArchivist().set_message( - "message:download", mess_dict, expire=False - ) + RedisArchivist().set_message(self.MSG, mess_dict) self.move_to_archive(vid_dict) mess_dict = { - "status": "message:download", + "status": self.MSG, "level": "info", "title": "Completed", "message": "", } - RedisArchivist().set_message("message:download", mess_dict, 10) + RedisArchivist().set_message(self.MSG, mess_dict, expire=10) self._delete_from_pending(youtube_id) # post processing @@ -212,16 +213,15 @@ class VideoDownloader: pending.get_channels() self.video_overwrites = pending.video_overwrites - @staticmethod - def add_pending(): + def add_pending(self): """add pending videos to download queue""" mess_dict = { - "status": "message:download", + "status": self.MSG, "level": "info", "title": "Looking for videos to download", "message": "Scanning your download queue.", } - RedisArchivist().set_message("message:download", mess_dict) + RedisArchivist().set_message(self.MSG, mess_dict, expire=True) pending = PendingList() pending.get_download() to_add = [i["youtube_id"] for i in pending.all_pending] @@ -229,18 +229,17 @@ class VideoDownloader: # there is nothing pending print("download queue is empty") mess_dict = { - "status": "message:download", + "status": self.MSG, "level": "error", "title": "Download queue is empty", "message": "Add some videos to the queue first.", } - RedisArchivist().set_message("message:download", mess_dict) + RedisArchivist().set_message(self.MSG, mess_dict, expire=True) return RedisQueue().add_list(to_add) - @staticmethod - def _progress_hook(response): + def _progress_hook(self, response): """process the progress_hooks from yt_dlp""" # title path = os.path.split(response["filename"])[-1][12:] @@ -257,12 +256,12 @@ class VideoDownloader: except KeyError: message = "processing" mess_dict = { - "status": "message:download", + "status": self.MSG, "level": "info", "title": title, "message": message, } - RedisArchivist().set_message("message:download", mess_dict) + RedisArchivist().set_message(self.MSG, mess_dict, expire=True) def _build_obs(self): """collection to build all obs passed to yt-dlp""" diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index ed937ae..9e213b5 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -98,7 +98,7 @@ class PostData: origin, new_view = self.exec_val.split(":") key = f"{self.current_user}:view:{origin}" print(f"change view: {key} to {new_view}") - RedisArchivist().set_message(key, {"status": new_view}, expire=False) + RedisArchivist().set_message(key, {"status": new_view}) return {"success": True} def _change_grid(self): @@ -109,7 +109,7 @@ class PostData: key = f"{self.current_user}:grid_items" print(f"change grid items: {grid_items}") - RedisArchivist().set_message(key, {"status": grid_items}, expire=False) + RedisArchivist().set_message(key, {"status": grid_items}) return {"success": True} @staticmethod @@ -135,7 +135,7 @@ class PostData: running = download_pending.delay() task_id = running.id print(f"{task_id}: set task id") - RedisArchivist().set_message("dl_queue_id", task_id, expire=False) + RedisArchivist().set_message("dl_queue_id", task_id) return {"success": True} def _queue_handler(self): @@ -187,11 +187,11 @@ class PostData: sort_order = {"status": self.exec_val} if self.exec_val in ["asc", "desc"]: RedisArchivist().set_message( - f"{self.current_user}:sort_order", sort_order, expire=False + f"{self.current_user}:sort_order", sort_order ) else: RedisArchivist().set_message( - f"{self.current_user}:sort_by", sort_order, expire=False + f"{self.current_user}:sort_by", sort_order ) return {"success": True} @@ -200,7 +200,7 @@ class PostData: key = f"{self.current_user}:hide_watched" message = {"status": bool(int(self.exec_val))} print(f"toggle {key}: {message}") - RedisArchivist().set_message(key, message, expire=False) + RedisArchivist().set_message(key, message) return {"success": True} def _show_subed_only(self): @@ -208,7 +208,7 @@ class PostData: key = f"{self.current_user}:show_subed_only" message = {"status": bool(int(self.exec_val))} print(f"toggle {key}: {message}") - RedisArchivist().set_message(key, message, expire=False) + RedisArchivist().set_message(key, message) return {"success": True} def _dlnow(self): @@ -218,7 +218,7 @@ class PostData: running = download_single.delay(youtube_id=youtube_id) task_id = running.id print("set task id: " + task_id) - RedisArchivist().set_message("dl_queue_id", task_id, expire=False) + RedisArchivist().set_message("dl_queue_id", task_id) return {"success": True} def _show_ignored_only(self): @@ -227,7 +227,7 @@ class PostData: key = f"{self.current_user}:show_ignored_only" value = {"status": show_value} print(f"Filter download view ignored only: {show_value}") - RedisArchivist().set_message(key, value, expire=False) + RedisArchivist().set_message(key, value) return {"success": True} def _forget_ignore(self): diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 4761cd0..eea9852 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -153,6 +153,7 @@ class YoutubeChannel(YouTubeItem): es_path = False index_name = "ta_channel" yt_base = "https://www.youtube.com/channel/" + msg = "message:playlistscan" def __init__(self, youtube_id): super().__init__(youtube_id) @@ -252,12 +253,12 @@ class YoutubeChannel(YouTubeItem): self.get_from_es() channel_name = self.json_data["channel_name"] mess_dict = { - "status": "message:playlistscan", + "status": self.msg, "level": "info", "title": "Looking for playlists", "message": f"{channel_name}: Scanning channel in progress", } - RedisArchivist().set_message("message:playlistscan", mess_dict) + RedisArchivist().set_message(self.msg, mess_dict, expire=True) self.get_all_playlists() if not self.all_playlists: print(f"{self.youtube_id}: no playlists found.") @@ -272,12 +273,12 @@ class YoutubeChannel(YouTubeItem): """send notification""" channel_name = self.json_data["channel_name"] mess_dict = { - "status": "message:playlistscan", + "status": self.msg, "level": "info", "title": f"{channel_name}: Scanning channel for playlists", "message": f"Progress: {idx + 1}/{len(self.all_playlists)}", } - RedisArchivist().set_message("message:playlistscan", mess_dict) + RedisArchivist().set_message(self.msg, mess_dict, expire=True) print("add playlist: " + playlist[1]) @staticmethod diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index c6f1caa..520be78 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -310,4 +310,4 @@ def reindex_old_documents(): handler = Reindex() handler.check_outdated() handler.reindex() - RedisArchivist().set_message("last_reindex", handler.now, expire=False) + RedisArchivist().set_message("last_reindex", handler.now) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 86d44a0..cc8d65b 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -339,7 +339,7 @@ class SponsorBlock: sb_id = RedisArchivist().get_message(key) if not sb_id["status"]: sb_id = {"status": randomizor(32)} - RedisArchivist().set_message(key, sb_id, expire=False) + RedisArchivist().set_message(key, sb_id) return sb_id diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 020e637..a964288 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -99,7 +99,7 @@ class AppConfig: self.config[config_dict][config_value] = to_write updated.append((config_value, to_write)) - RedisArchivist().set_message("config", self.config, expire=False) + RedisArchivist().set_message("config", self.config) return updated @staticmethod @@ -111,7 +111,7 @@ class AppConfig: message = {"status": value} redis_key = f"{user_id}:{key}" - RedisArchivist().set_message(redis_key, message, expire=False) + RedisArchivist().set_message(redis_key, message) def get_colors(self): """overwrite config if user has set custom values""" @@ -151,7 +151,7 @@ class AppConfig: needs_update = True if needs_update: - RedisArchivist().set_message("config", redis_config, expire=False) + RedisArchivist().set_message("config", redis_config) class ScheduleBuilder: @@ -165,6 +165,7 @@ class ScheduleBuilder: "run_backup": "0 18 0", } CONFIG = ["check_reindex_days", "run_backup_rotate"] + MSG = "message:setting" def __init__(self): self.config = AppConfig().config @@ -180,25 +181,27 @@ class ScheduleBuilder: except ValueError: print(f"failed: {key} {value}") mess_dict = { - "status": "message:setting", + "status": self.MSG, "level": "error", "title": "Scheduler update failed.", "message": "Invalid schedule input", } - RedisArchivist().set_message("message:setting", mess_dict) + RedisArchivist().set_message( + self.MSG, mess_dict, expire=True + ) return redis_config["scheduler"][key] = to_write if key in self.CONFIG and value: redis_config["scheduler"][key] = int(value) - RedisArchivist().set_message("config", redis_config, expire=False) + RedisArchivist().set_message("config", redis_config) mess_dict = { - "status": "message:setting", + "status": self.MSG, "level": "info", "title": "Scheduler changed.", "message": "Please restart container for changes to take effect", } - RedisArchivist().set_message("message:setting", mess_dict) + RedisArchivist().set_message(self.MSG, mess_dict, expire=True) def value_builder(self, key, value): """validate single cron form entry and return cron dict""" diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 544a8bd..c684de7 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -34,7 +34,7 @@ class RedisArchivist(RedisBase): "setting", ] - def set_message(self, key, message, path=".", expire=True): + def set_message(self, key, message, path=".", expire=False): """write new message to redis""" self.conn.execute_command( "JSON.SET", self.NAME_SPACE + key, path, json.dumps(message) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index b8f419b..8f78d4d 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -48,7 +48,7 @@ def update_subscribed(): "title": "Rescanning channels and playlists.", "message": "Looking for new videos.", } - RedisArchivist().set_message("message:rescan", message) + RedisArchivist().set_message("message:rescan", message, expire=True) have_lock = False my_lock = RedisArchivist().get_lock("rescan") @@ -108,13 +108,14 @@ def download_single(youtube_id): try: have_lock = my_lock.acquire(blocking=False) if have_lock: + key = "message:download" mess_dict = { - "status": "message:download", + "status": key, "level": "info", "title": "Download single video", "message": "processing", } - RedisArchivist().set_message("message:download", mess_dict) + RedisArchivist().set_message(key, mess_dict, expire=True) VideoDownloader().run_queue() else: print("Download queue already running.") @@ -196,7 +197,7 @@ def kill_dl(task_id): "title": "Canceling download process", "message": "Canceling download queue now.", } - RedisArchivist().set_message("message:download", mess_dict) + RedisArchivist().set_message("message:download", mess_dict, expire=True) @shared_task @@ -245,13 +246,14 @@ def subscribe_to(url_str): channel_id_sub, channel_subscribed=True ) # notify for channels + key = "message:subchannel" message = { - "status": "message:subchannel", + "status": key, "level": "info", "title": "Subscribing to Channels", "message": f"Processing {counter} of {len(to_subscribe_list)}", } - RedisArchivist().set_message("message:subchannel", message=message) + RedisArchivist().set_message(key, message=message, expire=True) counter = counter + 1 @@ -260,13 +262,14 @@ def index_channel_playlists(channel_id): """add all playlists of channel to index""" channel = YoutubeChannel(channel_id) # notify + key = "message:playlistscan" mess_dict = { - "status": "message:playlistscan", + "status": key, "level": "info", "title": "Looking for playlists", "message": f'Scanning channel "{channel.youtube_id}" in progress', } - RedisArchivist().set_message("message:playlistscan", mess_dict) + RedisArchivist().set_message(key, mess_dict, expire=True) channel.index_channel_playlists() diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index f99d3ae..bb30e8d 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -394,14 +394,15 @@ class DownloadView(ArchivistResultsView): youtube_ids = UrlListParser(url_str).process_list() except ValueError: # failed to process + key = "message:add" print(f"failed to parse: {url_str}") mess_dict = { - "status": "message:add", + "status": key, "level": "error", "title": "Failed to extract links.", "message": "Not a video, channel or playlist ID or URL", } - RedisArchivist().set_message("message:add", mess_dict) + RedisArchivist().set_message(key, mess_dict, expire=True) return redirect("downloads") print(youtube_ids) @@ -512,13 +513,14 @@ class ChannelView(ArchivistResultsView): """handle http post requests""" subscribe_form = SubscribeToChannelForm(data=request.POST) if subscribe_form.is_valid(): + key = "message:subchannel" message = { - "status": "message:subchannel", + "status": key, "level": "info", "title": "Subscribing to Channels", "message": "Parsing form data", } - RedisArchivist().set_message("message:subchannel", message=message) + RedisArchivist().set_message(key, message=message, expire=True) url_str = request.POST.get("subscribe") print(url_str) subscribe_to.delay(url_str) @@ -659,15 +661,14 @@ class PlaylistView(ArchivistResultsView): if subscribe_form.is_valid(): url_str = request.POST.get("subscribe") print(url_str) + key = "message:subplaylist" message = { - "status": "message:subplaylist", + "status": key, "level": "info", "title": "Subscribing to Playlists", "message": "Parsing form data", } - RedisArchivist().set_message( - "message:subplaylist", message=message - ) + RedisArchivist().set_message(key, message=message, expire=True) subscribe_to.delay(url_str) sleep(1) @@ -847,15 +848,14 @@ class SettingsView(View): valid = handler.validate() if not valid: handler.revoke() + key = "message:setting" message = { - "status": "message:setting", + "status": key, "level": "error", "title": "Cookie import failed", "message": "", } - RedisArchivist().set_message( - "message:setting", message=message - ) + RedisArchivist().set_message(key, message=message, expire=True) else: handler.revoke()