From fc311a7b7e0f076aa16173d505948d0cfbf9d70c Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 21 Mar 2022 11:14:44 +0700 Subject: [PATCH] refactor validate_playlists and move into DownloadPostProcess class --- .../home/src/download/yt_dlp_handler.py | 93 ++++++++++--------- tubearchivist/home/tasks.py | 3 +- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index a9848b5..74cd6f4 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -30,11 +30,17 @@ class DownloadPostProcess: def __init__(self, download): self.download = download self.now = int(datetime.now().strftime("%s")) + self.pending = False def run(self): """run all functions""" + self.pending = PendingList() + self.pending.get_download() + self.pending.get_channels() + self.pending.get_indexed() self.auto_delete_all() self.auto_delete_overwrites() + self.validate_playlists() def auto_delete_all(self): """handle auto delete""" @@ -52,10 +58,7 @@ class DownloadPostProcess: def auto_delete_overwrites(self): """handle per channel auto delete from overwrites""" - pending = PendingList() - pending.get_download() - pending.get_channels() - for channel_id, value in pending.channel_overwrites.items(): + for channel_id, value in self.pending.channel_overwrites.items(): if "autodelete_days" in value: autodelete_days = value.get("autodelete_days") print(f"{channel_id}: delete older than {autodelete_days}d") @@ -88,6 +91,47 @@ class DownloadPostProcess: pending.parse_url_list() pending.add_to_pending(status="ignore") + def validate_playlists(self): + """look for playlist needing to update""" + for id_c, channel_id in enumerate(self.download.channels): + playlists = YoutubeChannel(channel_id).get_indexed_playlists() + all_channel_playlist = [i["playlist_id"] for i in playlists] + self._validate_channel_playlist(all_channel_playlist, id_c) + + def _validate_channel_playlist(self, all_channel_playlist, id_c): + """scan channel for playlist needing update""" + all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos] + for id_p, playlist_id in enumerate(all_channel_playlist): + playlist = YoutubePlaylist(playlist_id) + playlist.all_youtube_ids = all_youtube_ids + playlist.build_json(scrape=True) + if not playlist.json_data: + playlist.deactivate() + + playlist.add_vids_to_playlist() + playlist.upload_to_es() + self._notify_playlist_progress(all_channel_playlist, id_c, id_p) + + def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p): + """notify to UI""" + title = ( + "Processing playlists for channels: " + + f"{id_c + 1}/{len(self.download.channels)}" + ) + message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}" + mess_dict = { + "status": "message:download", + "level": "info", + "title": title, + "message": message, + } + if id_p + 1 == len(all_channel_playlist): + RedisArchivist().set_message( + "message:download", mess_dict, expire=4 + ) + else: + RedisArchivist().set_message("message:download", mess_dict) + class VideoDownloader: """ @@ -132,6 +176,7 @@ class VideoDownloader: self._delete_from_pending(youtube_id) # post processing + self._add_subscribed_channels() DownloadPostProcess(self).run() @staticmethod @@ -338,43 +383,3 @@ class VideoDownloader: self.channels.add(channel_id) return - - def validate_playlists(self): - """look for playlist needing to update""" - print("sync playlists") - self._add_subscribed_channels() - pending = PendingList() - pending.get_download() - pending.get_indexed() - all_youtube_ids = [i["youtube_id"] for i in pending.all_videos] - for id_c, channel_id in enumerate(self.channels): - playlists = YoutubeChannel(channel_id).get_indexed_playlists() - all_playlist_ids = [i["playlist_id"] for i in playlists] - for id_p, playlist_id in enumerate(all_playlist_ids): - playlist = YoutubePlaylist(playlist_id) - playlist.all_youtube_ids = all_youtube_ids - playlist.build_json(scrape=True) - if not playlist.json_data: - playlist.deactivate() - - playlist.add_vids_to_playlist() - playlist.upload_to_es() - - # notify - title = ( - "Processing playlists for channels: " - + f"{id_c + 1}/{len(self.channels)}" - ) - message = f"Progress: {id_p + 1}/{len(all_playlist_ids)}" - mess_dict = { - "status": "message:download", - "level": "info", - "title": title, - "message": message, - } - if id_p + 1 == len(all_playlist_ids): - RedisArchivist().set_message( - "message:download", mess_dict, expire=4 - ) - else: - RedisArchivist().set_message("message:download", mess_dict) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index b013742..6d89983 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -88,7 +88,6 @@ def download_pending(): downloader = VideoDownloader() downloader.add_pending() downloader.run_queue() - downloader.validate_playlists() else: print("Did not acquire download lock.") @@ -276,8 +275,8 @@ def index_channel_playlists(channel_id): return handler = PendingList() - handler.get_indexed() handler.get_download() + handler.get_indexed() all_youtube_ids = [i["youtube_id"] for i in handler.all_videos] for idx, (playlist_id, playlist_title) in enumerate(all_playlists):