tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

major refactor ThumbManager

This commit is contained in:
simon 2022-08-10 21:03:54 +07:00
parent b76fa69396
commit fb4d6b7be3
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
9 changed files with 279 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tubearchivist/api/src/search_processor.py
View File

@ -74,7 +74,7 @@ class SearchProcess:
media_url = urllib.parse.quote(video_dict["media_url"])
vid_last_refresh = date_praser(video_dict["vid_last_refresh"])
published = date_praser(video_dict["published"])
vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
channel = self._process_channel(video_dict["channel"])
if "subtitles" in video_dict:
@ -113,7 +113,7 @@ class SearchProcess:
def _process_download(self, download_dict):
"""run on single download item"""
video_id = download_dict["youtube_id"]
vid_thumb_url = ThumbManager().vid_thumb_path(video_id)
vid_thumb_url = ThumbManager(video_id).vid_thumb_path()
published = date_praser(download_dict["published"])
download_dict.update(

11
tubearchivist/home/src/download/queue.py
View File

@ -161,10 +161,7 @@ class PendingList(PendingIndex):
self._parse_channel(entry["url"])
elif entry["type"] == "playlist":
self._parse_playlist(entry["url"])
new_thumbs = PlaylistSubscription().process_url_str(
[entry], subscribed=False
)
ThumbManager().download_playlist(new_thumbs)
PlaylistSubscription().process_url_str([entry], subscribed=False)
else:
raise ValueError(f"invalid url_type: {entry}")
@ -198,7 +195,6 @@ class PendingList(PendingIndex):
self.get_channels()
bulk_list = []
thumb_handler = ThumbManager()
for idx, youtube_id in enumerate(self.missing_videos):
video_details = self.get_youtube_details(youtube_id)
if not video_details:
@ -209,8 +205,9 @@ class PendingList(PendingIndex):
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(video_details))
thumb_needed = [(youtube_id, video_details["vid_thumb_url"])]
thumb_handler.download_vid(thumb_needed)
url = video_details["vid_thumb_url"]
ThumbManager(youtube_id).download_video_thumb(url)
self._notify_add(idx)
if bulk_list:

14
tubearchivist/home/src/download/subscriptions.py
View File

@ -5,6 +5,7 @@ Functionality:
"""
from home.src.download import queue # partial import
from home.src.download.thumbnails import ThumbManager
from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import IndexPaginate
from home.src.index.channel import YoutubeChannel
@ -129,11 +130,9 @@ class PlaylistSubscription:
all_indexed = IndexPaginate("ta_video", data).get_results()
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
new_thumbs = []
for idx, playlist in enumerate(new_playlists):
url_type = playlist["type"]
playlist_id = playlist["url"]
if not url_type == "playlist":
if not playlist["type"] == "playlist":
print(f"{playlist_id} not a playlist, skipping...")
continue
@ -144,8 +143,11 @@ class PlaylistSubscription:
playlist_h.upload_to_es()
playlist_h.add_vids_to_playlist()
self.channel_validate(playlist_h.json_data["playlist_channel_id"])
thumb = playlist_h.json_data["playlist_thumbnail"]
new_thumbs.append((playlist_id, thumb))
url = playlist_h.json_data["playlist_thumbnail"]
thumb = ThumbManager(playlist_id, item_type="playlist")
thumb.download_playlist_thumb(url)
# notify
message = {
"status": "message:subplaylist",
@ -157,8 +159,6 @@ class PlaylistSubscription:
"message:subplaylist", message=message, expire=True
)
return new_thumbs
@staticmethod
def channel_validate(channel_id):
"""make sure channel of playlist is there"""

485
tubearchivist/home/src/download/thumbnails.py
View File

@ -6,136 +6,64 @@ functionality:
import base64
import os
from collections import Counter
from io import BytesIO
from time import sleep
import requests
from home.src.download import queue # partial import
from home.src.download import subscriptions # partial import
from home.src.es.connect import IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.ta_redis import RedisArchivist
from mutagen.mp4 import MP4, MP4Cover
from PIL import Image, ImageFile, ImageFilter
ImageFile.LOAD_TRUNCATED_IMAGES = True
class ThumbManager:
"""handle thumbnails related functions"""
class ThumbManagerBase:
"""base class for thumbnail management"""
CONFIG = AppConfig().config
MEDIA_DIR = CONFIG["application"]["videos"]
CACHE_DIR = CONFIG["application"]["cache_dir"]
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
def get_all_thumbs(self):
"""get all video artwork already downloaded"""
all_thumb_folders = ignore_filelist(os.listdir(self.VIDEO_DIR))
all_thumbs = []
for folder in all_thumb_folders:
folder_path = os.path.join(self.VIDEO_DIR, folder)
if os.path.isfile(folder_path):
self.update_path(folder)
all_thumbs.append(folder_path)
continue
# raise exemption here in a future version
# raise FileExistsError("video cache dir has files inside")
def __init__(self, item_id, item_type, fallback=False):
self.item_id = item_id
self.item_type = item_type
self.fallback = fallback
all_folder_thumbs = ignore_filelist(os.listdir(folder_path))
all_thumbs.extend(all_folder_thumbs)
def download_raw(self, url):
"""download thumbnail for video"""
return all_thumbs
for i in range(3):
try:
response = requests.get(url, stream=True)
if response.ok:
return Image.open(response.raw)
if response.status_code == 404:
return self.get_fallback()
def update_path(self, file_name):
"""reorganize thumbnails into folders as update path from v0.0.5"""
folder_name = file_name[0].lower()
folder_path = os.path.join(self.VIDEO_DIR, folder_name)
old_file = os.path.join(self.VIDEO_DIR, file_name)
new_file = os.path.join(folder_path, file_name)
os.makedirs(folder_path, exist_ok=True)
os.rename(old_file, new_file)
except ConnectionError:
print(f"{self.item_id}: retry thumbnail download {url}")
sleep((i + 1) ** i)
def get_needed_thumbs(self, missing_only=False):
"""get a list of all missing thumbnails"""
all_thumbs = self.get_all_thumbs()
return False
pending = queue.PendingList()
pending.get_download()
pending.get_indexed()
def get_fallback(self):
"""get fallback thumbnail if not available"""
if self.fallback:
img_raw = Image.open(self.fallback)
return img_raw
needed_thumbs = []
for video in pending.all_videos:
youtube_id = video["youtube_id"]
thumb_url = video["vid_thumb_url"]
if missing_only:
if youtube_id + ".jpg" not in all_thumbs:
needed_thumbs.append((youtube_id, thumb_url))
else:
needed_thumbs.append((youtube_id, thumb_url))
for video in pending.all_pending + pending.all_ignored:
youtube_id = video["youtube_id"]
thumb_url = video["vid_thumb_url"]
if missing_only:
if youtube_id + ".jpg" not in all_thumbs:
needed_thumbs.append((youtube_id, thumb_url))
else:
needed_thumbs.append((youtube_id, thumb_url))
return needed_thumbs
def get_missing_channels(self):
"""get all channel artwork"""
all_channel_art = os.listdir(self.CHANNEL_DIR)
files = [i[0:24] for i in all_channel_art]
cached_channel_ids = [k for (k, v) in Counter(files).items() if v > 1]
channel_sub = subscriptions.ChannelSubscription()
channels = channel_sub.get_channels(subscribed_only=False)
missing_channels = []
for channel in channels:
channel_id = channel["channel_id"]
if channel_id not in cached_channel_ids:
channel_banner = channel["channel_banner_url"]
channel_thumb = channel["channel_thumb_url"]
missing_channels.append(
(channel_id, channel_thumb, channel_banner)
)
return missing_channels
def get_missing_playlists(self):
"""get all missing playlist artwork"""
all_downloaded = ignore_filelist(os.listdir(self.PLAYLIST_DIR))
all_ids_downloaded = [i.replace(".jpg", "") for i in all_downloaded]
playlist_sub = subscriptions.PlaylistSubscription()
playlists = playlist_sub.get_playlists(subscribed_only=False)
missing_playlists = []
for playlist in playlists:
playlist_id = playlist["playlist_id"]
if playlist_id not in all_ids_downloaded:
playlist_thumb = playlist["playlist_thumbnail"]
missing_playlists.append((playlist_id, playlist_thumb))
return missing_playlists
def get_raw_img(self, img_url, thumb_type):
"""get raw image from youtube and handle 404"""
try:
app_root = self.CONFIG["application"]["app_root"]
except KeyError:
# lazy keyerror fix to not have to deal with a strange startup
# racing contition between the threads in HomeConfig.ready()
app_root = "/app"
app_root = self.CONFIG["application"]["app_root"]
default_map = {
"video": os.path.join(
app_root, "static/img/default-video-thumb.jpg"
),
"playlist": os.path.join(
app_root, "static/img/default-video-thumb.jpg"
),
"icon": os.path.join(
app_root, "static/img/default-channel-icon.jpg"
),
@ -143,116 +71,134 @@ class ThumbManager:
app_root, "static/img/default-channel-banner.jpg"
),
}
if img_url:
try:
response = requests.get(img_url, stream=True)
except ConnectionError:
sleep(5)
response = requests.get(img_url, stream=True)
if not response.ok and not response.status_code == 404:
print("retry thumbnail download for " + img_url)
sleep(5)
response = requests.get(img_url, stream=True)
else:
response = False
if not response or response.status_code == 404:
# use default
img_raw = Image.open(default_map[thumb_type])
else:
# use response
img_obj = response.raw
img_raw = Image.open(img_obj)
img_raw = Image.open(default_map[self.item_type])
return img_raw
def download_vid(self, missing_thumbs, notify=True):
"""download all missing thumbnails from list"""
print(f"downloading {len(missing_thumbs)} thumbnails")
for idx, (youtube_id, thumb_url) in enumerate(missing_thumbs):
folder_path = os.path.join(self.VIDEO_DIR, youtube_id[0].lower())
thumb_path = os.path.join(
self.CACHE_DIR, self.vid_thumb_path(youtube_id)
)
os.makedirs(folder_path, exist_ok=True)
img_raw = self.get_raw_img(thumb_url, "video")
class ThumbManager(ThumbManagerBase):
"""handle thumbnails related functions"""
width, height = img_raw.size
if not width / height == 16 / 9:
new_height = width / 16 * 9
offset = (height - new_height) / 2
img_raw = img_raw.crop((0, offset, width, height - offset))
img_raw.convert("RGB").save(thumb_path)
def __init__(self, item_id, item_type="video", fallback=False):
super().__init__(item_id, item_type, fallback=fallback)
progress = f"{idx + 1}/{len(missing_thumbs)}"
if notify:
mess_dict = {
"status": "message:add",
"level": "info",
"title": "Processing Videos",
"message": "Downloading Thumbnails, Progress: " + progress,
}
if idx + 1 == len(missing_thumbs):
expire = 4
else:
expire = True
def download(self, url):
"""download thumbnail"""
print(f"{self.item_id}: download {self.item_type} thumbnail")
if self.item_type == "video":
self.download_video_thumb(url)
elif self.item_type == "channel":
self.download_channel_art(url)
elif self.item_type == "playlist":
self.download_playlist_thumb(url)
RedisArchivist().set_message(
"message:add", mess_dict, expire=expire
)
def delete(self):
"""delete thumbnail file"""
print(f"{self.item_id}: delete {self.item_type} thumbnail")
if self.item_type == "video":
self.delete_video_thumb()
elif self.item_type == "channel":
self.delete_channel_thumb()
elif self.item_type == "playlist":
self.delete_playlist_thumb()
if idx + 1 % 25 == 0:
print("thumbnail progress: " + progress)
def download_video_thumb(self, url, skip_existing=False):
"""pass url for video thumbnail"""
folder_path = os.path.join(self.VIDEO_DIR, self.item_id[0].lower())
thumb_path = self.vid_thumb_path(absolute=True)
def download_chan(self, missing_channels):
"""download needed artwork for channels"""
print(f"downloading {len(missing_channels)} channel artwork")
for channel in missing_channels:
channel_id, channel_thumb, channel_banner = channel
if skip_existing and os.path.exists(thumb_path):
return
thumb_path = os.path.join(
self.CHANNEL_DIR, channel_id + "_thumb.jpg"
)
img_raw = self.get_raw_img(channel_thumb, "icon")
img_raw.convert("RGB").save(thumb_path)
os.makedirs(folder_path, exist_ok=True)
img_raw = self.download_raw(url)
width, height = img_raw.size
banner_path = os.path.join(
self.CHANNEL_DIR, channel_id + "_banner.jpg"
)
img_raw = self.get_raw_img(channel_banner, "banner")
img_raw.convert("RGB").save(banner_path)
if not width / height == 16 / 9:
new_height = width / 16 * 9
offset = (height - new_height) / 2
img_raw = img_raw.crop((0, offset, width, height - offset))
mess_dict = {
"status": "message:download",
"level": "info",
"title": "Processing Channels",
"message": "Downloading Channel Art.",
}
key = "message:download"
RedisArchivist().set_message(key, mess_dict, expire=True)
img_raw.convert("RGB").save(thumb_path)
def download_playlist(self, missing_playlists):
"""download needed artwork for playlists"""
print(f"downloading {len(missing_playlists)} playlist artwork")
for playlist in missing_playlists:
playlist_id, playlist_thumb_url = playlist
thumb_path = os.path.join(self.PLAYLIST_DIR, playlist_id + ".jpg")
img_raw = self.get_raw_img(playlist_thumb_url, "video")
img_raw.convert("RGB").save(thumb_path)
def vid_thumb_path(self, absolute=False):
"""build expected path for video thumbnail from youtube_id"""
folder_name = self.item_id[0].lower()
folder_path = os.path.join("videos", folder_name)
thumb_path = os.path.join(folder_path, f"{self.item_id}.jpg")
if absolute:
thumb_path = os.path.join(self.CACHE_DIR, thumb_path)
mess_dict = {
"status": "message:download",
"level": "info",
"title": "Processing Playlists",
"message": "Downloading Playlist Art.",
}
key = "message:download"
RedisArchivist().set_message(key, mess_dict, expire=True)
return thumb_path
def get_base64_blur(self, youtube_id):
def download_channel_art(self, urls, skip_existing=False):
"""pass tuple of channel thumbnails"""
channel_thumb, channel_banner = urls
self._download_channel_thumb(channel_thumb, skip_existing)
self._download_channel_banner(channel_banner, skip_existing)
def _download_channel_thumb(self, channel_thumb, skip_existing):
"""download channel thumbnail"""
thumb_path = os.path.join(
self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg"
)
self.item_type = "icon"
if skip_existing and os.path.exists(thumb_path):
return
img_raw = self.download_raw(channel_thumb)
img_raw.convert("RGB").save(thumb_path)
def _download_channel_banner(self, channel_banner, skip_existing):
"""download channel banner"""
banner_path = os.path.join(
self.CHANNEL_DIR, self.item_id + "_banner.jpg"
)
self.item_type = "banner"
if skip_existing and os.path.exists(banner_path):
return
img_raw = self.download_raw(channel_banner)
img_raw.convert("RGB").save(banner_path)
def download_playlist_thumb(self, url, skip_existing=False):
"""pass thumbnail url"""
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
if skip_existing and os.path.exists(thumb_path):
return
img_raw = self.download_raw(url)
img_raw.convert("RGB").save(thumb_path)
def delete_video_thumb(self):
"""delete video thumbnail if exists"""
thumb_path = self.vid_thumb_path()
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
if os.path.exists(to_delete):
os.remove(to_delete)
def delete_channel_thumb(self):
"""delete all artwork of channel"""
thumb = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_thumb.jpg")
banner = os.path.join(self.CHANNEL_DIR, f"{self.item_id}_banner.jpg")
if os.path.exists(thumb):
os.remove(thumb)
if os.path.exists(banner):
os.remove(banner)
def delete_playlist_thumb(self):
"""delete playlist thumbnail"""
thumb_path = os.path.join(self.PLAYLIST_DIR, f"{self.item_id}.jpg")
if os.path.exists(thumb_path):
os.remove(thumb_path)
def get_vid_base64_blur(self):
"""return base64 encoded placeholder"""
img_path = self.vid_thumb_path(youtube_id)
file_path = os.path.join(self.CACHE_DIR, img_path)
file_path = os.path.join(self.CACHE_DIR, self.vid_thumb_path())
img_raw = Image.open(file_path)
img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
img_blur = img_raw.filter(ImageFilter.BLUR)
@ -264,40 +210,109 @@ class ThumbManager:
return data_url
@staticmethod
def vid_thumb_path(youtube_id):
"""build expected path for video thumbnail from youtube_id"""
folder_name = youtube_id[0].lower()
folder_path = os.path.join("videos", folder_name)
thumb_path = os.path.join(folder_path, youtube_id + ".jpg")
return thumb_path
def delete_vid_thumb(self, youtube_id):
"""delete video thumbnail if exists"""
thumb_path = self.vid_thumb_path(youtube_id)
to_delete = os.path.join(self.CACHE_DIR, thumb_path)
if os.path.exists(to_delete):
os.remove(to_delete)
class ValidatorCallback:
"""handle callback validate thumbnails page by page"""
def delete_chan_thumb(self, channel_id):
"""delete all artwork of channel"""
thumb = os.path.join(self.CHANNEL_DIR, channel_id + "_thumb.jpg")
banner = os.path.join(self.CHANNEL_DIR, channel_id + "_banner.jpg")
if os.path.exists(thumb):
os.remove(thumb)
if os.path.exists(banner):
os.remove(banner)
def __init__(self, source, index_name):
self.source = source
self.index_name = index_name
def cleanup_downloaded(self):
"""find downloaded thumbnails without video indexed"""
all_thumbs = self.get_all_thumbs()
all_indexed = self.get_needed_thumbs()
all_needed_thumbs = [i[0] + ".jpg" for i in all_indexed]
for thumb in all_thumbs:
if thumb not in all_needed_thumbs:
# cleanup
youtube_id = thumb.rstrip(".jpg")
self.delete_vid_thumb(youtube_id)
def run(self):
"""run the task for page"""
print(f"{self.index_name}: validate artwork")
if self.index_name == "ta_video":
self._validate_videos()
elif self.index_name == "ta_channel":
self._validate_channels()
elif self.index_name == "ta_playlist":
self._validate_playlists()
def _validate_videos(self):
"""check if video thumbnails are correct"""
for video in self.source:
url = video["_source"]["vid_thumb_url"]
handler = ThumbManager(video["_source"]["youtube_id"])
handler.download_video_thumb(url, skip_existing=True)
def _validate_channels(self):
"""check if all channel artwork is there"""
for channel in self.source:
urls = (
channel["_source"]["channel_thumb_url"],
channel["_source"]["channel_banner_url"],
)
handler = ThumbManager(channel["_source"]["channel_id"])
handler.download_channel_art(urls, skip_existing=True)
def _validate_playlists(self):
"""check if all playlist artwork is there"""
for playlist in self.source:
url = playlist["_source"]["playlist_thumbnail"]
handler = ThumbManager(playlist["_source"]["playlist_id"])
handler.download_playlist_thumb(url, skip_existing=True)
class ThumbValidator:
"""validate thumbnails"""
def download_missing(self):
"""download all missing artwork"""
self.download_missing_videos()
self.download_missing_channels()
self.download_missing_playlists()
def download_missing_videos(self):
"""get all missing video thumbnails"""
data = {
"query": {"term": {"active": {"value": True}}},
"sort": [{"youtube_id": {"order": "asc"}}],
"_source": ["vid_thumb_url", "youtube_id"],
}
paginate = IndexPaginate(
"ta_video", data, size=5000, callback=ValidatorCallback
)
_ = paginate.get_results()
def download_missing_channels(self):
"""get all missing channel thumbnails"""
data = {
"query": {"term": {"channel_active": {"value": True}}},
"sort": [{"channel_id": {"order": "asc"}}],
"_source": {
"excludes": ["channel_description", "channel_overwrites"]
},
}
paginate = IndexPaginate(
"ta_channel", data, callback=ValidatorCallback
)
_ = paginate.get_results()
def download_missing_playlists(self):
"""get all missing playlist artwork"""
data = {
"query": {"term": {"playlist_active": {"value": True}}},
"sort": [{"playlist_id": {"order": "asc"}}],
"_source": ["playlist_id", "playlist_thumbnail"],
}
paginate = IndexPaginate(
"ta_playlist", data, callback=ValidatorCallback
)
_ = paginate.get_results()
class ThumbFilesystem:
"""filesystem tasks for thumbnails"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
MEDIA_DIR = CONFIG["application"]["videos"]
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
def sync(self):
"""embed thumbnails to mediafiles"""
video_list = self.get_thumb_list()
self._embed_thumbs(video_list)
def get_thumb_list(self):
"""get list of mediafiles and matching thumbnails"""
@ -307,10 +322,10 @@ class ThumbManager:
video_list = []
for video in pending.all_videos:
youtube_id = video["youtube_id"]
video_id = video["youtube_id"]
media_url = os.path.join(self.MEDIA_DIR, video["media_url"])
thumb_path = os.path.join(
self.CACHE_DIR, self.vid_thumb_path(youtube_id)
self.CACHE_DIR, ThumbManager(video_id).vid_thumb_path()
)
video_list.append(
{
@ -322,7 +337,7 @@ class ThumbManager:
return video_list
@staticmethod
def write_all_thumbs(video_list):
def _embed_thumbs(video_list):
"""rewrite the thumbnail into media file"""
counter = 1
@ -340,15 +355,3 @@ class ThumbManager:
if counter % 50 == 0:
print(f"thumbnail write progress {counter}/{len(video_list)}")
counter = counter + 1
def validate_thumbnails():
"""check if all thumbnails are there and organized correctly"""
handler = ThumbManager()
thumbs_to_download = handler.get_needed_thumbs(missing_only=True)
handler.download_vid(thumbs_to_download)
missing_channels = handler.get_missing_channels()
handler.download_chan(missing_channels)
missing_playlists = handler.get_missing_playlists()
handler.download_playlist(missing_playlists)
handler.cleanup_downloaded()

4
tubearchivist/home/src/frontend/searching.py
View File

@ -119,7 +119,7 @@ class SearchHandler:
if "vid_thumb_url" in hit_keys:
youtube_id = hit["source"]["youtube_id"]
thumb_path = ThumbManager().vid_thumb_path(youtube_id)
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
hit["source"]["vid_thumb_url"] = thumb_path
if "channel_last_refresh" in hit_keys:
@ -138,7 +138,7 @@ class SearchHandler:
if "subtitle_fragment_id" in hit_keys:
youtube_id = hit["source"]["youtube_id"]
thumb_path = ThumbManager().vid_thumb_path(youtube_id)
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
return hit

8
tubearchivist/home/src/index/channel.py
View File

@ -192,11 +192,11 @@ class YoutubeChannel(YouTubeItem):
def get_channel_art(self):
"""download channel art for new channels"""
channel_id = self.youtube_id
channel_thumb = self.json_data["channel_thumb_url"]
channel_banner = self.json_data["channel_banner_url"]
ThumbManager().download_chan(
[(channel_id, channel_thumb, channel_banner)]
urls = (
self.json_data["channel_thumb_url"],
self.json_data["channel_banner_url"],
)
ThumbManager(channel_id, item_type="channel").download(urls)
def sync_to_videos(self):
"""sync new channel_dict to all videos of channel"""

18
tubearchivist/home/src/index/playlist.py
View File

@ -81,12 +81,10 @@ class YoutubePlaylist(YouTubeItem):
self.all_members = all_members
@staticmethod
def get_playlist_art():
def get_playlist_art(self):
"""download artwork of playlist"""
thumbnails = ThumbManager()
missing_playlists = thumbnails.get_missing_playlists()
thumbnails.download_playlist(missing_playlists)
url = self.json_data["playlist_thumbnail"]
ThumbManager(self.youtube_id, item_type="playlist").download(url)
def add_vids_to_playlist(self):
"""sync the playlist id to videos"""
@ -145,17 +143,15 @@ class YoutubePlaylist(YouTubeItem):
previous_item = False
else:
previous_item = all_entries[current_idx - 1]
prev_thumb = ThumbManager().vid_thumb_path(
previous_item["youtube_id"]
)
previous_item["vid_thumb"] = prev_thumb
prev_id = previous_item["youtube_id"]
previous_item["vid_thumb"] = ThumbManager(prev_id).vid_thumb_path()
if current_idx == len(all_entries) - 1:
next_item = False
else:
next_item = all_entries[current_idx + 1]
next_thumb = ThumbManager().vid_thumb_path(next_item["youtube_id"])
next_item["vid_thumb"] = next_thumb
next_id = next_item["youtube_id"]
next_item["vid_thumb"] = ThumbManager(next_id).vid_thumb_path()
self.nav = {
"playlist_meta": {

8
tubearchivist/home/src/index/reindex.py
View File

@ -181,10 +181,10 @@ class Reindex:
video.upload_to_es()
thumb_handler = ThumbManager()
thumb_handler.delete_vid_thumb(youtube_id)
to_download = (youtube_id, video.json_data["vid_thumb_url"])
thumb_handler.download_vid([to_download], notify=False)
thumb_handler = ThumbManager(youtube_id)
thumb_handler.delete_video_thumb()
thumb_handler.download_video_thumb(video.json_data["vid_thumb_url"])
return
@staticmethod

14
tubearchivist/home/tasks.py
View File

@ -15,7 +15,7 @@ from home.src.download.subscriptions import (
ChannelSubscription,
PlaylistSubscription,
)
from home.src.download.thumbnails import ThumbManager, validate_thumbnails
from home.src.download.thumbnails import ThumbFilesystem, ThumbValidator
from home.src.download.yt_dlp_handler import VideoDownloader
from home.src.es.index_setup import backup_all_indexes, restore_from_backup
from home.src.index.channel import YoutubeChannel
@ -201,21 +201,19 @@ def kill_dl(task_id):
def rescan_filesystem():
"""check the media folder for mismatches"""
scan_filesystem()
validate_thumbnails()
ThumbValidator().download_missing()
@shared_task(name="thumbnail_check")
def thumbnail_check():
"""validate thumbnails"""
validate_thumbnails()
ThumbValidator().download_missing()
@shared_task
def re_sync_thumbs():
"""sync thumbnails to mediafiles"""
handler = ThumbManager()
video_list = handler.get_thumb_list()
handler.write_all_thumbs(video_list)
ThumbFilesystem().sync()
@shared_task
@ -226,9 +224,7 @@ def subscribe_to(url_str):
for item in to_subscribe_list:
to_sub_id = item["url"]
if item["type"] == "playlist":
new_thumbs = PlaylistSubscription().process_url_str([item])
if new_thumbs:
ThumbManager().download_playlist(new_thumbs)
PlaylistSubscription().process_url_str([item])
continue
if item["type"] == "video":