mirror of
https://github.com/tubearchivist/tubearchivist-frontend.git
synced 2024-11-04 19:30:13 +00:00
refactor validate_playlists and move into DownloadPostProcess class
This commit is contained in:
parent
f803c5298b
commit
fc311a7b7e
@ -30,11 +30,17 @@ class DownloadPostProcess:
|
|||||||
def __init__(self, download):
|
def __init__(self, download):
|
||||||
self.download = download
|
self.download = download
|
||||||
self.now = int(datetime.now().strftime("%s"))
|
self.now = int(datetime.now().strftime("%s"))
|
||||||
|
self.pending = False
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""run all functions"""
|
"""run all functions"""
|
||||||
|
self.pending = PendingList()
|
||||||
|
self.pending.get_download()
|
||||||
|
self.pending.get_channels()
|
||||||
|
self.pending.get_indexed()
|
||||||
self.auto_delete_all()
|
self.auto_delete_all()
|
||||||
self.auto_delete_overwrites()
|
self.auto_delete_overwrites()
|
||||||
|
self.validate_playlists()
|
||||||
|
|
||||||
def auto_delete_all(self):
|
def auto_delete_all(self):
|
||||||
"""handle auto delete"""
|
"""handle auto delete"""
|
||||||
@ -52,10 +58,7 @@ class DownloadPostProcess:
|
|||||||
|
|
||||||
def auto_delete_overwrites(self):
|
def auto_delete_overwrites(self):
|
||||||
"""handle per channel auto delete from overwrites"""
|
"""handle per channel auto delete from overwrites"""
|
||||||
pending = PendingList()
|
for channel_id, value in self.pending.channel_overwrites.items():
|
||||||
pending.get_download()
|
|
||||||
pending.get_channels()
|
|
||||||
for channel_id, value in pending.channel_overwrites.items():
|
|
||||||
if "autodelete_days" in value:
|
if "autodelete_days" in value:
|
||||||
autodelete_days = value.get("autodelete_days")
|
autodelete_days = value.get("autodelete_days")
|
||||||
print(f"{channel_id}: delete older than {autodelete_days}d")
|
print(f"{channel_id}: delete older than {autodelete_days}d")
|
||||||
@ -88,6 +91,47 @@ class DownloadPostProcess:
|
|||||||
pending.parse_url_list()
|
pending.parse_url_list()
|
||||||
pending.add_to_pending(status="ignore")
|
pending.add_to_pending(status="ignore")
|
||||||
|
|
||||||
|
def validate_playlists(self):
|
||||||
|
"""look for playlist needing to update"""
|
||||||
|
for id_c, channel_id in enumerate(self.download.channels):
|
||||||
|
playlists = YoutubeChannel(channel_id).get_indexed_playlists()
|
||||||
|
all_channel_playlist = [i["playlist_id"] for i in playlists]
|
||||||
|
self._validate_channel_playlist(all_channel_playlist, id_c)
|
||||||
|
|
||||||
|
def _validate_channel_playlist(self, all_channel_playlist, id_c):
|
||||||
|
"""scan channel for playlist needing update"""
|
||||||
|
all_youtube_ids = [i["youtube_id"] for i in self.pending.all_videos]
|
||||||
|
for id_p, playlist_id in enumerate(all_channel_playlist):
|
||||||
|
playlist = YoutubePlaylist(playlist_id)
|
||||||
|
playlist.all_youtube_ids = all_youtube_ids
|
||||||
|
playlist.build_json(scrape=True)
|
||||||
|
if not playlist.json_data:
|
||||||
|
playlist.deactivate()
|
||||||
|
|
||||||
|
playlist.add_vids_to_playlist()
|
||||||
|
playlist.upload_to_es()
|
||||||
|
self._notify_playlist_progress(all_channel_playlist, id_c, id_p)
|
||||||
|
|
||||||
|
def _notify_playlist_progress(self, all_channel_playlist, id_c, id_p):
|
||||||
|
"""notify to UI"""
|
||||||
|
title = (
|
||||||
|
"Processing playlists for channels: "
|
||||||
|
+ f"{id_c + 1}/{len(self.download.channels)}"
|
||||||
|
)
|
||||||
|
message = f"Progress: {id_p + 1}/{len(all_channel_playlist)}"
|
||||||
|
mess_dict = {
|
||||||
|
"status": "message:download",
|
||||||
|
"level": "info",
|
||||||
|
"title": title,
|
||||||
|
"message": message,
|
||||||
|
}
|
||||||
|
if id_p + 1 == len(all_channel_playlist):
|
||||||
|
RedisArchivist().set_message(
|
||||||
|
"message:download", mess_dict, expire=4
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
RedisArchivist().set_message("message:download", mess_dict)
|
||||||
|
|
||||||
|
|
||||||
class VideoDownloader:
|
class VideoDownloader:
|
||||||
"""
|
"""
|
||||||
@ -132,6 +176,7 @@ class VideoDownloader:
|
|||||||
self._delete_from_pending(youtube_id)
|
self._delete_from_pending(youtube_id)
|
||||||
|
|
||||||
# post processing
|
# post processing
|
||||||
|
self._add_subscribed_channels()
|
||||||
DownloadPostProcess(self).run()
|
DownloadPostProcess(self).run()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@ -338,43 +383,3 @@ class VideoDownloader:
|
|||||||
self.channels.add(channel_id)
|
self.channels.add(channel_id)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|
||||||
def validate_playlists(self):
|
|
||||||
"""look for playlist needing to update"""
|
|
||||||
print("sync playlists")
|
|
||||||
self._add_subscribed_channels()
|
|
||||||
pending = PendingList()
|
|
||||||
pending.get_download()
|
|
||||||
pending.get_indexed()
|
|
||||||
all_youtube_ids = [i["youtube_id"] for i in pending.all_videos]
|
|
||||||
for id_c, channel_id in enumerate(self.channels):
|
|
||||||
playlists = YoutubeChannel(channel_id).get_indexed_playlists()
|
|
||||||
all_playlist_ids = [i["playlist_id"] for i in playlists]
|
|
||||||
for id_p, playlist_id in enumerate(all_playlist_ids):
|
|
||||||
playlist = YoutubePlaylist(playlist_id)
|
|
||||||
playlist.all_youtube_ids = all_youtube_ids
|
|
||||||
playlist.build_json(scrape=True)
|
|
||||||
if not playlist.json_data:
|
|
||||||
playlist.deactivate()
|
|
||||||
|
|
||||||
playlist.add_vids_to_playlist()
|
|
||||||
playlist.upload_to_es()
|
|
||||||
|
|
||||||
# notify
|
|
||||||
title = (
|
|
||||||
"Processing playlists for channels: "
|
|
||||||
+ f"{id_c + 1}/{len(self.channels)}"
|
|
||||||
)
|
|
||||||
message = f"Progress: {id_p + 1}/{len(all_playlist_ids)}"
|
|
||||||
mess_dict = {
|
|
||||||
"status": "message:download",
|
|
||||||
"level": "info",
|
|
||||||
"title": title,
|
|
||||||
"message": message,
|
|
||||||
}
|
|
||||||
if id_p + 1 == len(all_playlist_ids):
|
|
||||||
RedisArchivist().set_message(
|
|
||||||
"message:download", mess_dict, expire=4
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
RedisArchivist().set_message("message:download", mess_dict)
|
|
||||||
|
@ -88,7 +88,6 @@ def download_pending():
|
|||||||
downloader = VideoDownloader()
|
downloader = VideoDownloader()
|
||||||
downloader.add_pending()
|
downloader.add_pending()
|
||||||
downloader.run_queue()
|
downloader.run_queue()
|
||||||
downloader.validate_playlists()
|
|
||||||
else:
|
else:
|
||||||
print("Did not acquire download lock.")
|
print("Did not acquire download lock.")
|
||||||
|
|
||||||
@ -276,8 +275,8 @@ def index_channel_playlists(channel_id):
|
|||||||
return
|
return
|
||||||
|
|
||||||
handler = PendingList()
|
handler = PendingList()
|
||||||
handler.get_indexed()
|
|
||||||
handler.get_download()
|
handler.get_download()
|
||||||
|
handler.get_indexed()
|
||||||
all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
|
all_youtube_ids = [i["youtube_id"] for i in handler.all_videos]
|
||||||
|
|
||||||
for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
|
for idx, (playlist_id, playlist_title) in enumerate(all_playlists):
|
||||||
|
Loading…
Reference in New Issue
Block a user