diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index e026dba..77c345a 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -40,7 +40,7 @@ from home.tasks import ( run_restore_backup, subscribe_to, ) -from rest_framework import permissions +from rest_framework import permissions, status from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, @@ -462,12 +462,26 @@ class PlaylistApiListView(ApiBaseView): search_base = "ta_playlist/_search/" permission_classes = [AdminWriteOnly] + valid_playlist_type = ["regular", "custom"] def get(self, request): """handle get request""" - self.data.update( - {"sort": [{"playlist_name.keyword": {"order": "asc"}}]} - ) + playlist_type = request.GET.get("playlist_type", None) + query = {"sort": [{"playlist_name.keyword": {"order": "asc"}}]} + if playlist_type is not None: + if playlist_type not in self.valid_playlist_type: + message = f"invalid playlist_type {playlist_type}" + return Response({"message": message}, status=400) + + query.update( + { + "query": { + "term": {"playlist_type": {"value": playlist_type}} + }, + } + ) + + self.data.update(query) self.get_document_list(request) return Response(self.response) @@ -511,6 +525,7 @@ class PlaylistApiView(ApiBaseView): search_base = "ta_playlist/_doc/" permission_classes = [AdminWriteOnly] + valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"] def get(self, request, playlist_id): # pylint: disable=unused-argument @@ -518,6 +533,27 @@ class PlaylistApiView(ApiBaseView): self.get_document(playlist_id) return Response(self.response, status=self.status_code) + def post(self, request, playlist_id): + """post to custom playlist to add a video to list""" + playlist = YoutubePlaylist(playlist_id) + if not playlist.is_custom_playlist(): + message = f"playlist with ID {playlist_id} is not custom" + return Response({"message": message}, status=400) + + action = request.data.get("action") + if action not in self.valid_custom_actions: + message = f"invalid action: {action}" + return Response({"message": message}, status=400) + + video_id = request.data.get("video_id") + if action == "create": + playlist.add_video_to_playlist(video_id) + else: + hide = UserConfig(request.user.id).get_value("hide_watched") + playlist.move_video(video_id, action, hide_watched=hide) + + return Response({"success": True}, status=status.HTTP_201_CREATED) + def delete(self, request, playlist_id): """delete playlist""" print(f"{playlist_id}: delete playlist") diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index b387fb4..16d43d2 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -8,6 +8,7 @@ import os from time import sleep from django.core.management.base import BaseCommand, CommandError +from home.src.es.connect import ElasticWrap from home.src.es.index_setup import ElasitIndexWrap from home.src.es.snapshot import ElasticSnapshot from home.src.ta.config import AppConfig, ReleaseVersion @@ -44,6 +45,7 @@ class Command(BaseCommand): self._mig_index_setup() self._mig_snapshot_check() self._mig_move_users_to_es() + self._mig_custom_playlist() def _sync_redis_state(self): """make sure redis gets new config.json values""" @@ -242,3 +244,36 @@ class Command(BaseCommand): " ✓ Settings for all users migrated to ES" ) ) + + def _mig_custom_playlist(self): + """migration for custom playlist""" + self.stdout.write("[MIGRATION] custom playlist") + data = { + "query": { + "bool": {"must_not": [{"exists": {"field": "playlist_type"}}]} + }, + "script": {"source": "ctx._source['playlist_type'] = 'regular'"}, + } + path = "ta_playlist/_update_by_query" + response, status_code = ElasticWrap(path).post(data=data) + if status_code == 200: + updated = response.get("updated", 0) + if updated: + self.stdout.write( + self.style.SUCCESS( + f" ✓ {updated} playlist_type updated in ta_playlist" + ) + ) + else: + self.stdout.write( + self.style.SUCCESS( + " no playlist_type needed updating in ta_playlist" + ) + ) + return + + message = " 🗙 ta_playlist playlist_type update failed" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(response) + sleep(60) + raise CommandError(message) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 896f603..a2698fd 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -75,7 +75,7 @@ class ThumbManagerBase: app_root, "static/img/default-video-thumb.jpg" ), "playlist": os.path.join( - app_root, "static/img/default-video-thumb.jpg" + app_root, "static/img/default-playlist-thumb.jpg" ), "icon": os.path.join( app_root, "static/img/default-channel-icon.jpg" @@ -202,7 +202,18 @@ class ThumbManager(ThumbManagerBase): if skip_existing and os.path.exists(thumb_path): return - img_raw = self.download_raw(url) + img_raw = ( + self.download_raw(url) + if not isinstance(url, str) or url.startswith("http") + else Image.open(os.path.join(self.CACHE_DIR, url)) + ) + width, height = img_raw.size + + if not width / height == 16 / 9: + new_height = width / 16 * 9 + offset = (height - new_height) / 2 + img_raw = img_raw.crop((0, offset, width, height - offset)) + img_raw = img_raw.resize((336, 189)) img_raw.convert("RGB").save(thumb_path) def delete_video_thumb(self): diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index b1c8fa7..d95a297 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -62,8 +62,8 @@ class YtWrap: """make extract request""" try: response = yt_dlp.YoutubeDL(self.obs).extract_info(url) - except cookiejar.LoadError: - print("cookie file is invalid") + except cookiejar.LoadError as err: + print(f"cookie file is invalid: {err}") return False except yt_dlp.utils.ExtractorError as err: print(f"{url}: failed to extract with message: {err}, continue...") diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 5044456..f8fcb79 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -105,8 +105,8 @@ class ApplicationSettingsForm(forms.Form): COOKIE_IMPORT_CHOICES = [ ("", "-- change cookie settings"), - ("0", "disable cookie"), - ("1", "enable cookie"), + ("0", "remove cookie"), + ("1", "import cookie"), ] subscriptions_channel_size = forms.IntegerField( @@ -265,6 +265,20 @@ class SubscribeToPlaylistForm(forms.Form): ) +class CreatePlaylistForm(forms.Form): + """text area form to create a single custom playlist""" + + create = forms.CharField( + label="Or create custom playlist", + widget=forms.Textarea( + attrs={ + "rows": 1, + "placeholder": "Input playlist name", + } + ), + ) + + class ChannelOverwriteForm(forms.Form): """custom overwrites for channel settings""" diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 3bfaec1..08166f7 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -213,6 +213,7 @@ class YoutubeChannel(YouTubeItem): all_playlists = self.get_indexed_playlists() for playlist in all_playlists: playlist_id = playlist["playlist_id"] + playlist = YoutubePlaylist(playlist_id) YoutubePlaylist(playlist_id).delete_metadata() def delete_channel(self): diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py index 84e6cdd..91b18ff 100644 --- a/tubearchivist/home/src/index/comments.py +++ b/tubearchivist/home/src/index/comments.py @@ -115,6 +115,9 @@ class Comments: time_text = time_text_datetime.strftime(format_string) + if not comment.get("author"): + comment["author"] = comment.get("author_id", "Unknown") + cleaned_comment = { "comment_id": comment["id"], "comment_text": comment["text"].replace("\xa0", ""), diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py index 656da4a..196a884 100644 --- a/tubearchivist/home/src/index/playlist.py +++ b/tubearchivist/home/src/index/playlist.py @@ -66,6 +66,7 @@ class YoutubePlaylist(YouTubeItem): "playlist_thumbnail": playlist_thumbnail, "playlist_description": self.youtube_meta["description"] or False, "playlist_last_refresh": int(datetime.now().timestamp()), + "playlist_type": "regular", } def get_entries(self, playlistend=False): @@ -178,6 +179,7 @@ class YoutubePlaylist(YouTubeItem): def delete_metadata(self): """delete metadata for playlist""" + self.delete_videos_metadata() script = ( "ctx._source.playlist.removeAll(" + "Collections.singleton(params.playlist)) " @@ -195,6 +197,30 @@ class YoutubePlaylist(YouTubeItem): _, _ = ElasticWrap("ta_video/_update_by_query").post(data) self.del_in_es() + def is_custom_playlist(self): + self.get_from_es() + return self.json_data["playlist_type"] == "custom" + + def delete_videos_metadata(self, channel_id=None): + """delete video metadata for a specific channel""" + self.get_from_es() + playlist = self.json_data["playlist_entries"] + i = 0 + while i < len(playlist): + video_id = playlist[i]["youtube_id"] + video = YoutubeVideo(video_id) + video.get_from_es() + if ( + channel_id is None + or video.json_data["channel"]["channel_id"] == channel_id + ): + playlist.pop(i) + self.remove_playlist_from_video(video_id) + i -= 1 + i += 1 + self.set_playlist_thumbnail() + self.upload_to_es() + def delete_videos_playlist(self): """delete playlist with all videos""" print(f"{self.youtube_id}: delete playlist") @@ -208,3 +234,159 @@ class YoutubePlaylist(YouTubeItem): YoutubeVideo(youtube_id).delete_media_file() self.delete_metadata() + + def create(self, name): + self.json_data = { + "playlist_id": self.youtube_id, + "playlist_active": False, + "playlist_name": name, + "playlist_last_refresh": int(datetime.now().timestamp()), + "playlist_entries": [], + "playlist_type": "custom", + "playlist_channel": None, + "playlist_channel_id": None, + "playlist_description": False, + "playlist_thumbnail": False, + "playlist_subscribed": False, + } + self.upload_to_es() + self.get_playlist_art() + return True + + def add_video_to_playlist(self, video_id): + self.get_from_es() + video_metadata = self.get_video_metadata(video_id) + video_metadata["idx"] = len(self.json_data["playlist_entries"]) + + if not self.playlist_entries_contains(video_id): + self.json_data["playlist_entries"].append(video_metadata) + self.json_data["playlist_last_refresh"] = int( + datetime.now().timestamp() + ) + self.set_playlist_thumbnail() + self.upload_to_es() + video = YoutubeVideo(video_id) + video.get_from_es() + if "playlist" not in video.json_data: + video.json_data["playlist"] = [] + video.json_data["playlist"].append(self.youtube_id) + video.upload_to_es() + return True + + def remove_playlist_from_video(self, video_id): + video = YoutubeVideo(video_id) + video.get_from_es() + if video.json_data is not None and "playlist" in video.json_data: + video.json_data["playlist"].remove(self.youtube_id) + video.upload_to_es() + + def move_video(self, video_id, action, hide_watched=False): + self.get_from_es() + video_index = self.get_video_index(video_id) + playlist = self.json_data["playlist_entries"] + item = playlist[video_index] + playlist.pop(video_index) + if action == "remove": + self.remove_playlist_from_video(item["youtube_id"]) + else: + if action == "up": + while True: + video_index = max(0, video_index - 1) + if ( + not hide_watched + or video_index == 0 + or ( + not self.get_video_is_watched( + playlist[video_index]["youtube_id"] + ) + ) + ): + break + elif action == "down": + while True: + video_index = min(len(playlist), video_index + 1) + if ( + not hide_watched + or video_index == len(playlist) + or ( + not self.get_video_is_watched( + playlist[video_index - 1]["youtube_id"] + ) + ) + ): + break + elif action == "top": + video_index = 0 + else: + video_index = len(playlist) + playlist.insert(video_index, item) + self.json_data["playlist_last_refresh"] = int( + datetime.now().timestamp() + ) + + for i, item in enumerate(playlist): + item["idx"] = i + + self.set_playlist_thumbnail() + self.upload_to_es() + + return True + + def del_video(self, video_id): + playlist = self.json_data["playlist_entries"] + + i = 0 + while i < len(playlist): + if video_id == playlist[i]["youtube_id"]: + playlist.pop(i) + self.set_playlist_thumbnail() + i -= 1 + i += 1 + + def get_video_index(self, video_id): + for i, child in enumerate(self.json_data["playlist_entries"]): + if child["youtube_id"] == video_id: + return i + return -1 + + def playlist_entries_contains(self, video_id): + return ( + len( + list( + filter( + lambda x: x["youtube_id"] == video_id, + self.json_data["playlist_entries"], + ) + ) + ) + > 0 + ) + + def get_video_is_watched(self, video_id): + video = YoutubeVideo(video_id) + video.get_from_es() + return video.json_data["player"]["watched"] + + def set_playlist_thumbnail(self): + playlist = self.json_data["playlist_entries"] + self.json_data["playlist_thumbnail"] = False + + for video in playlist: + url = ThumbManager(video["youtube_id"]).vid_thumb_path() + if url is not None: + self.json_data["playlist_thumbnail"] = url + break + self.get_playlist_art() + + def get_video_metadata(self, video_id): + video = YoutubeVideo(video_id) + video.get_from_es() + video_json_data = { + "youtube_id": video.json_data["youtube_id"], + "title": video.json_data["title"], + "uploader": video.json_data["channel"]["channel_name"], + "idx": 0, + "downloaded": "date_downloaded" in video.json_data + and video.json_data["date_downloaded"] > 0, + } + return video_json_data diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index fe4d986..10a5596 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -363,7 +363,10 @@ class Reindex(ReindexBase): self._get_all_videos() playlist = YoutubePlaylist(playlist_id) playlist.get_from_es() - if not playlist.json_data: + if ( + not playlist.json_data + or playlist.json_data["playlist_type"] == "custom" + ): return subscribed = playlist.json_data["playlist_subscribed"] diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index b57ad55..5697351 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -128,6 +128,10 @@ class YoutubeSubtitle: print(response.text) continue + if not response.text: + print(f"{self.video.youtube_id}: skip empty subtitle") + continue + parser = SubtitleParser(response.text, lang, source) parser.process() if not parser.all_cues: diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 1b258ae..41f0114 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -319,6 +319,8 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): playlist.json_data["playlist_entries"][idx].update( {"downloaded": False} ) + if playlist.json_data["playlist_type"] == "custom": + playlist.del_video(self.youtube_id) playlist.upload_to_es() def delete_subtitles(self, subtitles=False): diff --git a/tubearchivist/home/src/ta/urlparser.py b/tubearchivist/home/src/ta/urlparser.py index 04429fa..743af98 100644 --- a/tubearchivist/home/src/ta/urlparser.py +++ b/tubearchivist/home/src/ta/urlparser.py @@ -92,7 +92,7 @@ class Parser: item_type = "video" elif len_id_str == 24: item_type = "channel" - elif len_id_str in (34, 26, 18): + elif len_id_str in (34, 26, 18) or id_str.startswith("TA_playlist_"): item_type = "playlist" else: raise ValueError(f"not a valid id_str: {id_str}") diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index ab2d788..ea7baba 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -11,13 +11,18 @@ {% if request.user|has_group:"admin" or request.user.is_staff %}
Last refreshed: {{ playlist.playlist_last_refresh }}
- {% if playlist.playlist_subscribed %} - - {% else %} - - {% endif %} + {% if playlist.playlist_type != "custom" %} + {% if playlist.playlist_subscribed %} + + {% else %} + + {% endif %} + {% endif %}Last refreshed: {{ playlist_info.playlist_last_refresh }}
-Playlist: - {% if playlist_info.playlist_subscribed %} - {% if request.user|has_group:"admin" or request.user.is_staff %} - - {% endif %} - {% else %} - - {% endif %} -
- {% if playlist_info.playlist_active %} -Youtube: Active
- {% else %} -Youtube: Deactivated
+ {% if playlist_info.playlist_type != "custom" %} +Playlist: + {% if playlist_info.playlist_subscribed %} + {% if request.user|has_group:"admin" or request.user.is_staff %} + + {% endif %} + {% else %} + + {% endif %} +
+ {% if playlist_info.playlist_active %} +Youtube: Active
+ {% else %} +Youtube: Deactivated
+ {% endif %} {% endif %}Reindex scheduled
{% else %} {% endif %} @@ -138,15 +145,35 @@ {% endif %} {{ video.published }} | {{ video.player.duration_str }}Try going to the downloads page to start the scan and download tasks.
+ {% if playlist_info.playlist_type == "custom" %} +Try going to the home page to add videos to this playlist.
+ {% else %} +Try going to the downloads page to start the scan and download tasks.
+ {% endif %} {% endif %}