From fb4d6b7be3f4b76129829ee765de8c0a61135d94 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 10 Aug 2022 21:03:54 +0700 Subject: [PATCH] major refactor ThumbManager --- tubearchivist/api/src/search_processor.py | 4 +- tubearchivist/home/src/download/queue.py | 11 +- .../home/src/download/subscriptions.py | 14 +- tubearchivist/home/src/download/thumbnails.py | 485 +++++++++--------- tubearchivist/home/src/frontend/searching.py | 4 +- tubearchivist/home/src/index/channel.py | 8 +- tubearchivist/home/src/index/playlist.py | 18 +- tubearchivist/home/src/index/reindex.py | 8 +- tubearchivist/home/tasks.py | 14 +- 9 files changed, 279 insertions(+), 287 deletions(-) diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 6a1e2ddd..07102310 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -74,7 +74,7 @@ class SearchProcess: media_url = urllib.parse.quote(video_dict["media_url"]) vid_last_refresh = date_praser(video_dict["vid_last_refresh"]) published = date_praser(video_dict["published"]) - vid_thumb_url = ThumbManager().vid_thumb_path(video_id) + vid_thumb_url = ThumbManager(video_id).vid_thumb_path() channel = self._process_channel(video_dict["channel"]) if "subtitles" in video_dict: @@ -113,7 +113,7 @@ class SearchProcess: def _process_download(self, download_dict): """run on single download item""" video_id = download_dict["youtube_id"] - vid_thumb_url = ThumbManager().vid_thumb_path(video_id) + vid_thumb_url = ThumbManager(video_id).vid_thumb_path() published = date_praser(download_dict["published"]) download_dict.update( diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index b318b128..db7ae5d5 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -161,10 +161,7 @@ class PendingList(PendingIndex): self._parse_channel(entry["url"]) elif entry["type"] == "playlist": self._parse_playlist(entry["url"]) - new_thumbs = PlaylistSubscription().process_url_str( - [entry], subscribed=False - ) - ThumbManager().download_playlist(new_thumbs) + PlaylistSubscription().process_url_str([entry], subscribed=False) else: raise ValueError(f"invalid url_type: {entry}") @@ -198,7 +195,6 @@ class PendingList(PendingIndex): self.get_channels() bulk_list = [] - thumb_handler = ThumbManager() for idx, youtube_id in enumerate(self.missing_videos): video_details = self.get_youtube_details(youtube_id) if not video_details: @@ -209,8 +205,9 @@ class PendingList(PendingIndex): bulk_list.append(json.dumps(action)) bulk_list.append(json.dumps(video_details)) - thumb_needed = [(youtube_id, video_details["vid_thumb_url"])] - thumb_handler.download_vid(thumb_needed) + url = video_details["vid_thumb_url"] + ThumbManager(youtube_id).download_video_thumb(url) + self._notify_add(idx) if bulk_list: diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index d353ed06..7f8a1676 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -5,6 +5,7 @@ Functionality: """ from home.src.download import queue # partial import +from home.src.download.thumbnails import ThumbManager from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import IndexPaginate from home.src.index.channel import YoutubeChannel @@ -129,11 +130,9 @@ class PlaylistSubscription: all_indexed = IndexPaginate("ta_video", data).get_results() all_youtube_ids = [i["youtube_id"] for i in all_indexed] - new_thumbs = [] for idx, playlist in enumerate(new_playlists): - url_type = playlist["type"] playlist_id = playlist["url"] - if not url_type == "playlist": + if not playlist["type"] == "playlist": print(f"{playlist_id} not a playlist, skipping...") continue @@ -144,8 +143,11 @@ class PlaylistSubscription: playlist_h.upload_to_es() playlist_h.add_vids_to_playlist() self.channel_validate(playlist_h.json_data["playlist_channel_id"]) - thumb = playlist_h.json_data["playlist_thumbnail"] - new_thumbs.append((playlist_id, thumb)) + + url = playlist_h.json_data["playlist_thumbnail"] + thumb = ThumbManager(playlist_id, item_type="playlist") + thumb.download_playlist_thumb(url) + # notify message = { "status": "message:subplaylist", @@ -157,8 +159,6 @@ class PlaylistSubscription: "message:subplaylist", message=message, expire=True ) - return new_thumbs - @staticmethod def channel_validate(channel_id): """make sure channel of playlist is there""" diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index c317ee4d..8e6b8716 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -6,136 +6,64 @@ functionality: import base64 import os -from collections import Counter from io import BytesIO from time import sleep import requests from home.src.download import queue # partial import -from home.src.download import subscriptions # partial import +from home.src.es.connect import IndexPaginate from home.src.ta.config import AppConfig -from home.src.ta.helper import ignore_filelist -from home.src.ta.ta_redis import RedisArchivist from mutagen.mp4 import MP4, MP4Cover from PIL import Image, ImageFile, ImageFilter ImageFile.LOAD_TRUNCATED_IMAGES = True -class ThumbManager: - """handle thumbnails related functions""" +class ThumbManagerBase: + """base class for thumbnail management""" CONFIG = AppConfig().config - MEDIA_DIR = CONFIG["application"]["videos"] CACHE_DIR = CONFIG["application"]["cache_dir"] VIDEO_DIR = os.path.join(CACHE_DIR, "videos") CHANNEL_DIR = os.path.join(CACHE_DIR, "channels") PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists") - def get_all_thumbs(self): - """get all video artwork already downloaded""" - all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR)) - all_thumbs = [] - for folder in all_thumb_folders: - folder_path = os.path.join(self.VIDEO_DIR, folder) - if os.path.isfile(folder_path): - self.update_path(folder) - all_thumbs.append(folder_path) - continue - # raise exemption here in a future version - # raise FileExistsError("video cache dir has files inside") + def __init__(self, item_id, item_type, fallback=False): + self.item_id = item_id + self.item_type = item_type + self.fallback = fallback - all_folder_thumbs = ignore_filelist(os.listdir(folder_path)) - all_thumbs.extend(all_folder_thumbs) + def download_raw(self, url): + """download thumbnail for video""" - return all_thumbs + for i in range(3): + try: + response = requests.get(url, stream=True) + if response.ok: + return Image.open(response.raw) + if response.status_code == 404: + return self.get_fallback() - def update_path(self, file_name): - """reorganize thumbnails into folders as update path from v0.0.5""" - folder_name = file_name[0].lower() - folder_path = os.path.join(self.VIDEO_DIR, folder_name) - old_file = os.path.join(self.VIDEO_DIR, file_name) - new_file = os.path.join(folder_path, file_name) - os.makedirs(folder_path, exist_ok=True) - os.rename(old_file, new_file) + except ConnectionError: + print(f"{self.item_id}: retry thumbnail download {url}") + sleep((i + 1) ** i) - def get_needed_thumbs(self, missing_only=False): - """get a list of all missing thumbnails""" - all_thumbs = self.get_all_thumbs() + return False - pending = queue.PendingList() - pending.get_download() - pending.get_indexed() + def get_fallback(self): + """get fallback thumbnail if not available""" + if self.fallback: + img_raw = Image.open(self.fallback) + return img_raw - needed_thumbs = [] - for video in pending.all_videos: - youtube_id = video["youtube_id"] - thumb_url = video["vid_thumb_url"] - if missing_only: - if youtube_id + ".jpg" not in all_thumbs: - needed_thumbs.append((youtube_id, thumb_url)) - else: - needed_thumbs.append((youtube_id, thumb_url)) - - for video in pending.all_pending + pending.all_ignored: - youtube_id = video["youtube_id"] - thumb_url = video["vid_thumb_url"] - if missing_only: - if youtube_id + ".jpg" not in all_thumbs: - needed_thumbs.append((youtube_id, thumb_url)) - else: - needed_thumbs.append((youtube_id, thumb_url)) - - return needed_thumbs - - def get_missing_channels(self): - """get all channel artwork""" - all_channel_art = os.listdir(self.CHANNEL_DIR) - files = [i[0:24] for i in all_channel_art] - cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1] - channel_sub = subscriptions.ChannelSubscription() - channels = channel_sub.get_channels(subscribed_only=False) - - missing_channels = [] - for channel in channels: - channel_id = channel["channel_id"] - if channel_id not in cached_channel_ids: - channel_banner = channel["channel_banner_url"] - channel_thumb = channel["channel_thumb_url"] - missing_channels.append( - (channel_id, channel_thumb, channel_banner) - ) - - return missing_channels - - def get_missing_playlists(self): - """get all missing playlist artwork""" - all_downloaded = ignore_filelist(os.listdir(self.PLAYLIST_DIR)) - all_ids_downloaded = [i.replace(".jpg", "") for i in all_downloaded] - playlist_sub = subscriptions.PlaylistSubscription() - playlists = playlist_sub.get_playlists(subscribed_only=False) - - missing_playlists = [] - for playlist in playlists: - playlist_id = playlist["playlist_id"] - if playlist_id not in all_ids_downloaded: - playlist_thumb = playlist["playlist_thumbnail"] - missing_playlists.append((playlist_id, playlist_thumb)) - - return missing_playlists - - def get_raw_img(self, img_url, thumb_type): - """get raw image from youtube and handle 404""" - try: - app_root = self.CONFIG["application"]["app_root"] - except KeyError: - # lazy keyerror fix to not have to deal with a strange startup - # racing contition between the threads in HomeConfig.ready() - app_root = "/app" + app_root = self.CONFIG["application"]["app_root"] default_map = { "video": os.path.join( app_root, "static/img/default-video-thumb.jpg" ), + "playlist": os.path.join( + app_root, "static/img/default-video-thumb.jpg" + ), "icon": os.path.join( app_root, "static/img/default-channel-icon.jpg" ), @@ -143,116 +71,134 @@ class ThumbManager: app_root, "static/img/default-channel-banner.jpg" ), } - if img_url: - try: - response = requests.get(img_url, stream=True) - except ConnectionError: - sleep(5) - response = requests.get(img_url, stream=True) - if not response.ok and not response.status_code == 404: - print("retry thumbnail download for " + img_url) - sleep(5) - response = requests.get(img_url, stream=True) - else: - response = False - if not response or response.status_code == 404: - # use default - img_raw = Image.open(default_map[thumb_type]) - else: - # use response - img_obj = response.raw - img_raw = Image.open(img_obj) + + img_raw = Image.open(default_map[self.item_type]) return img_raw - def download_vid(self, missing_thumbs, notify=True): - """download all missing thumbnails from list""" - print(f"downloading {len(missing_thumbs)} thumbnails") - for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs): - folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower()) - thumb_path = os.path.join( - self.CACHE_DIR, self.vid_thumb_path(youtube_id) - ) - os.makedirs(folder_path, exist_ok=True) - img_raw = self.get_raw_img(thumb_url, "video") +class ThumbManager(ThumbManagerBase): + """handle thumbnails related functions""" - width, height = img_raw.size - if not width / height == 16 / 9: - new_height = width / 16 * 9 - offset = (height - new_height) / 2 - img_raw = img_raw.crop((0, offset, width, height - offset)) - img_raw.convert("RGB").save(thumb_path) + def __init__(self, item_id, item_type="video", fallback=False): + super().__init__(item_id, item_type, fallback=fallback) - progress = f"{idx + 1}/{len(missing_thumbs)}" - if notify: - mess_dict = { - "status": "message:add", - "level": "info", - "title": "Processing Videos", - "message": "Downloading Thumbnails, Progress: " + progress, - } - if idx + 1 == len(missing_thumbs): - expire = 4 - else: - expire = True + def download(self, url): + """download thumbnail""" + print(f"{self.item_id}: download {self.item_type} thumbnail") + if self.item_type == "video": + self.download_video_thumb(url) + elif self.item_type == "channel": + self.download_channel_art(url) + elif self.item_type == "playlist": + self.download_playlist_thumb(url) - RedisArchivist().set_message( - "message:add", mess_dict, expire=expire - ) + def delete(self): + """delete thumbnail file""" + print(f"{self.item_id}: delete {self.item_type} thumbnail") + if self.item_type == "video": + self.delete_video_thumb() + elif self.item_type == "channel": + self.delete_channel_thumb() + elif self.item_type == "playlist": + self.delete_playlist_thumb() - if idx + 1 % 25 == 0: - print("thumbnail progress: " + progress) + def download_video_thumb(self, url, skip_existing=False): + """pass url for video thumbnail""" + folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower()) + thumb_path = self.vid_thumb_path(absolute=True) - def download_chan(self, missing_channels): - """download needed artwork for channels""" - print(f"downloading {len(missing_channels)} channel artwork") - for channel in missing_channels: - channel_id, channel_thumb, channel_banner = channel + if skip_existing and os.path.exists(thumb_path): + return - thumb_path = os.path.join( - self.CHANNEL_DIR, channel_id + "_thumb.jpg" - ) - img_raw = self.get_raw_img(channel_thumb, "icon") - img_raw.convert("RGB").save(thumb_path) + os.makedirs(folder_path, exist_ok=True) + img_raw = self.download_raw(url) + width, height = img_raw.size - banner_path = os.path.join( - self.CHANNEL_DIR, channel_id + "_banner.jpg" - ) - img_raw = self.get_raw_img(channel_banner, "banner") - img_raw.convert("RGB").save(banner_path) + if not width / height == 16 / 9: + new_height = width / 16 * 9 + offset = (height - new_height) / 2 + img_raw = img_raw.crop((0, offset, width, height - offset)) - mess_dict = { - "status": "message:download", - "level": "info", - "title": "Processing Channels", - "message": "Downloading Channel Art.", - } - key = "message:download" - RedisArchivist().set_message(key, mess_dict, expire=True) + img_raw.convert("RGB").save(thumb_path) - def download_playlist(self, missing_playlists): - """download needed artwork for playlists""" - print(f"downloading {len(missing_playlists)} playlist artwork") - for playlist in missing_playlists: - playlist_id, playlist_thumb_url = playlist - thumb_path = os.path.join(self.PLAYLIST_DIR, playlist_id + ".jpg") - img_raw = self.get_raw_img(playlist_thumb_url, "video") - img_raw.convert("RGB").save(thumb_path) + def vid_thumb_path(self, absolute=False): + """build expected path for video thumbnail from youtube_id""" + folder_name = self.item_id[0].lower() + folder_path = os.path.join("videos", folder_name) + thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg") + if absolute: + thumb_path = os.path.join(self.CACHE_DIR, thumb_path) - mess_dict = { - "status": "message:download", - "level": "info", - "title": "Processing Playlists", - "message": "Downloading Playlist Art.", - } - key = "message:download" - RedisArchivist().set_message(key, mess_dict, expire=True) + return thumb_path - def get_base64_blur(self, youtube_id): + def download_channel_art(self, urls, skip_existing=False): + """pass tuple of channel thumbnails""" + channel_thumb, channel_banner = urls + self._download_channel_thumb(channel_thumb, skip_existing) + self._download_channel_banner(channel_banner, skip_existing) + + def _download_channel_thumb(self, channel_thumb, skip_existing): + """download channel thumbnail""" + + thumb_path = os.path.join( + self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg" + ) + self.item_type = "icon" + + if skip_existing and os.path.exists(thumb_path): + return + + img_raw = self.download_raw(channel_thumb) + img_raw.convert("RGB").save(thumb_path) + + def _download_channel_banner(self, channel_banner, skip_existing): + """download channel banner""" + + banner_path = os.path.join( + self.CHANNEL_DIR, self.item_id + "_banner.jpg" + ) + self.item_type = "banner" + if skip_existing and os.path.exists(banner_path): + return + + img_raw = self.download_raw(channel_banner) + img_raw.convert("RGB").save(banner_path) + + def download_playlist_thumb(self, url, skip_existing=False): + """pass thumbnail url""" + thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg") + if skip_existing and os.path.exists(thumb_path): + return + + img_raw = self.download_raw(url) + img_raw.convert("RGB").save(thumb_path) + + def delete_video_thumb(self): + """delete video thumbnail if exists""" + thumb_path = self.vid_thumb_path() + to_delete = os.path.join(self.CACHE_DIR, thumb_path) + if os.path.exists(to_delete): + os.remove(to_delete) + + def delete_channel_thumb(self): + """delete all artwork of channel""" + thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg") + banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg") + if os.path.exists(thumb): + os.remove(thumb) + if os.path.exists(banner): + os.remove(banner) + + def delete_playlist_thumb(self): + """delete playlist thumbnail""" + thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg") + if os.path.exists(thumb_path): + os.remove(thumb_path) + + def get_vid_base64_blur(self): """return base64 encoded placeholder""" - img_path = self.vid_thumb_path(youtube_id) - file_path = os.path.join(self.CACHE_DIR, img_path) + file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path()) img_raw = Image.open(file_path) img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20)) img_blur = img_raw.filter(ImageFilter.BLUR) @@ -264,40 +210,109 @@ class ThumbManager: return data_url - @staticmethod - def vid_thumb_path(youtube_id): - """build expected path for video thumbnail from youtube_id""" - folder_name = youtube_id[0].lower() - folder_path = os.path.join("videos", folder_name) - thumb_path = os.path.join(folder_path, youtube_id + ".jpg") - return thumb_path - def delete_vid_thumb(self, youtube_id): - """delete video thumbnail if exists""" - thumb_path = self.vid_thumb_path(youtube_id) - to_delete = os.path.join(self.CACHE_DIR, thumb_path) - if os.path.exists(to_delete): - os.remove(to_delete) +class ValidatorCallback: + """handle callback validate thumbnails page by page""" - def delete_chan_thumb(self, channel_id): - """delete all artwork of channel""" - thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg") - banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg") - if os.path.exists(thumb): - os.remove(thumb) - if os.path.exists(banner): - os.remove(banner) + def __init__(self, source, index_name): + self.source = source + self.index_name = index_name - def cleanup_downloaded(self): - """find downloaded thumbnails without video indexed""" - all_thumbs = self.get_all_thumbs() - all_indexed = self.get_needed_thumbs() - all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed] - for thumb in all_thumbs: - if thumb not in all_needed_thumbs: - # cleanup - youtube_id = thumb.rstrip(".jpg") - self.delete_vid_thumb(youtube_id) + def run(self): + """run the task for page""" + print(f"{self.index_name}: validate artwork") + if self.index_name == "ta_video": + self._validate_videos() + elif self.index_name == "ta_channel": + self._validate_channels() + elif self.index_name == "ta_playlist": + self._validate_playlists() + + def _validate_videos(self): + """check if video thumbnails are correct""" + for video in self.source: + url = video["_source"]["vid_thumb_url"] + handler = ThumbManager(video["_source"]["youtube_id"]) + handler.download_video_thumb(url, skip_existing=True) + + def _validate_channels(self): + """check if all channel artwork is there""" + for channel in self.source: + urls = ( + channel["_source"]["channel_thumb_url"], + channel["_source"]["channel_banner_url"], + ) + handler = ThumbManager(channel["_source"]["channel_id"]) + handler.download_channel_art(urls, skip_existing=True) + + def _validate_playlists(self): + """check if all playlist artwork is there""" + for playlist in self.source: + url = playlist["_source"]["playlist_thumbnail"] + handler = ThumbManager(playlist["_source"]["playlist_id"]) + handler.download_playlist_thumb(url, skip_existing=True) + + +class ThumbValidator: + """validate thumbnails""" + + def download_missing(self): + """download all missing artwork""" + self.download_missing_videos() + self.download_missing_channels() + self.download_missing_playlists() + + def download_missing_videos(self): + """get all missing video thumbnails""" + data = { + "query": {"term": {"active": {"value": True}}}, + "sort": [{"youtube_id": {"order": "asc"}}], + "_source": ["vid_thumb_url", "youtube_id"], + } + paginate = IndexPaginate( + "ta_video", data, size=5000, callback=ValidatorCallback + ) + _ = paginate.get_results() + + def download_missing_channels(self): + """get all missing channel thumbnails""" + data = { + "query": {"term": {"channel_active": {"value": True}}}, + "sort": [{"channel_id": {"order": "asc"}}], + "_source": { + "excludes": ["channel_description", "channel_overwrites"] + }, + } + paginate = IndexPaginate( + "ta_channel", data, callback=ValidatorCallback + ) + _ = paginate.get_results() + + def download_missing_playlists(self): + """get all missing playlist artwork""" + data = { + "query": {"term": {"playlist_active": {"value": True}}}, + "sort": [{"playlist_id": {"order": "asc"}}], + "_source": ["playlist_id", "playlist_thumbnail"], + } + paginate = IndexPaginate( + "ta_playlist", data, callback=ValidatorCallback + ) + _ = paginate.get_results() + + +class ThumbFilesystem: + """filesystem tasks for thumbnails""" + + CONFIG = AppConfig().config + CACHE_DIR = CONFIG["application"]["cache_dir"] + MEDIA_DIR = CONFIG["application"]["videos"] + VIDEO_DIR = os.path.join(CACHE_DIR, "videos") + + def sync(self): + """embed thumbnails to mediafiles""" + video_list = self.get_thumb_list() + self._embed_thumbs(video_list) def get_thumb_list(self): """get list of mediafiles and matching thumbnails""" @@ -307,10 +322,10 @@ class ThumbManager: video_list = [] for video in pending.all_videos: - youtube_id = video["youtube_id"] + video_id = video["youtube_id"] media_url = os.path.join(self.MEDIA_DIR, video["media_url"]) thumb_path = os.path.join( - self.CACHE_DIR, self.vid_thumb_path(youtube_id) + self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path() ) video_list.append( { @@ -322,7 +337,7 @@ class ThumbManager: return video_list @staticmethod - def write_all_thumbs(video_list): + def _embed_thumbs(video_list): """rewrite the thumbnail into media file""" counter = 1 @@ -340,15 +355,3 @@ class ThumbManager: if counter % 50 == 0: print(f"thumbnail write progress {counter}/{len(video_list)}") counter = counter + 1 - - -def validate_thumbnails(): - """check if all thumbnails are there and organized correctly""" - handler = ThumbManager() - thumbs_to_download = handler.get_needed_thumbs(missing_only=True) - handler.download_vid(thumbs_to_download) - missing_channels = handler.get_missing_channels() - handler.download_chan(missing_channels) - missing_playlists = handler.get_missing_playlists() - handler.download_playlist(missing_playlists) - handler.cleanup_downloaded() diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/home/src/frontend/searching.py index b14cc65d..ce3209ab 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/home/src/frontend/searching.py @@ -119,7 +119,7 @@ class SearchHandler: if "vid_thumb_url" in hit_keys: youtube_id = hit["source"]["youtube_id"] - thumb_path = ThumbManager().vid_thumb_path(youtube_id) + thumb_path = ThumbManager(youtube_id).vid_thumb_path() hit["source"]["vid_thumb_url"] = thumb_path if "channel_last_refresh" in hit_keys: @@ -138,7 +138,7 @@ class SearchHandler: if "subtitle_fragment_id" in hit_keys: youtube_id = hit["source"]["youtube_id"] - thumb_path = ThumbManager().vid_thumb_path(youtube_id) + thumb_path = ThumbManager(youtube_id).vid_thumb_path() hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}" return hit diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 400c5f5a..d41449ae 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -192,11 +192,11 @@ class YoutubeChannel(YouTubeItem): def get_channel_art(self): """download channel art for new channels""" channel_id = self.youtube_id - channel_thumb = self.json_data["channel_thumb_url"] - channel_banner = self.json_data["channel_banner_url"] - ThumbManager().download_chan( - [(channel_id, channel_thumb, channel_banner)] + urls = ( + self.json_data["channel_thumb_url"], + self.json_data["channel_banner_url"], ) + ThumbManager(channel_id, item_type="channel").download(urls) def sync_to_videos(self): """sync new channel_dict to all videos of channel""" diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py index 69dc5db5..7115f9d9 100644 --- a/tubearchivist/home/src/index/playlist.py +++ b/tubearchivist/home/src/index/playlist.py @@ -81,12 +81,10 @@ class YoutubePlaylist(YouTubeItem): self.all_members = all_members - @staticmethod - def get_playlist_art(): + def get_playlist_art(self): """download artwork of playlist""" - thumbnails = ThumbManager() - missing_playlists = thumbnails.get_missing_playlists() - thumbnails.download_playlist(missing_playlists) + url = self.json_data["playlist_thumbnail"] + ThumbManager(self.youtube_id, item_type="playlist").download(url) def add_vids_to_playlist(self): """sync the playlist id to videos""" @@ -145,17 +143,15 @@ class YoutubePlaylist(YouTubeItem): previous_item = False else: previous_item = all_entries[current_idx - 1] - prev_thumb = ThumbManager().vid_thumb_path( - previous_item["youtube_id"] - ) - previous_item["vid_thumb"] = prev_thumb + prev_id = previous_item["youtube_id"] + previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path() if current_idx == len(all_entries) - 1: next_item = False else: next_item = all_entries[current_idx + 1] - next_thumb = ThumbManager().vid_thumb_path(next_item["youtube_id"]) - next_item["vid_thumb"] = next_thumb + next_id = next_item["youtube_id"] + next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path() self.nav = { "playlist_meta": { diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index f0b88bd1..db231f89 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -181,10 +181,10 @@ class Reindex: video.upload_to_es() - thumb_handler = ThumbManager() - thumb_handler.delete_vid_thumb(youtube_id) - to_download = (youtube_id, video.json_data["vid_thumb_url"]) - thumb_handler.download_vid([to_download], notify=False) + thumb_handler = ThumbManager(youtube_id) + thumb_handler.delete_video_thumb() + thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"]) + return @staticmethod diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index ede17f26..b3da7621 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -15,7 +15,7 @@ from home.src.download.subscriptions import ( ChannelSubscription, PlaylistSubscription, ) -from home.src.download.thumbnails import ThumbManager, validate_thumbnails +from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.index_setup import backup_all_indexes, restore_from_backup from home.src.index.channel import YoutubeChannel @@ -201,21 +201,19 @@ def kill_dl(task_id): def rescan_filesystem(): """check the media folder for mismatches""" scan_filesystem() - validate_thumbnails() + ThumbValidator().download_missing() @shared_task(name="thumbnail_check") def thumbnail_check(): """validate thumbnails""" - validate_thumbnails() + ThumbValidator().download_missing() @shared_task def re_sync_thumbs(): """sync thumbnails to mediafiles""" - handler = ThumbManager() - video_list = handler.get_thumb_list() - handler.write_all_thumbs(video_list) + ThumbFilesystem().sync() @shared_task @@ -226,9 +224,7 @@ def subscribe_to(url_str): for item in to_subscribe_list: to_sub_id = item["url"] if item["type"] == "playlist": - new_thumbs = PlaylistSubscription().process_url_str([item]) - if new_thumbs: - ThumbManager().download_playlist(new_thumbs) + PlaylistSubscription().process_url_str([item]) continue if item["type"] == "video":