diff --git a/README.md b/README.md index e128104..ee642c5 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ There's dedicated user-contributed install steps under [docs/Installation.md](./ For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation. -Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27). +Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64. This project requires docker. Ensure it is installed and running on your system. diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 7fb48a0..8662ad0 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -20,6 +20,7 @@ from api.views import ( VideoApiView, VideoCommentView, VideoProgressView, + VideoSimilarView, VideoSponsorView, ) from django.urls import path @@ -47,6 +48,11 @@ urlpatterns = [ VideoCommentView.as_view(), name="api-video-comment", ), + path( + "video//similar/", + VideoSimilarView.as_view(), + name="api-video-similar", + ), path( "video//sponsor/", VideoSponsorView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index dab62e3..ea771f0 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -62,10 +62,13 @@ class ApiBaseView(APIView): } ) - def get_document_list(self, request): + def get_document_list(self, request, pagination=True): """get a list of results""" print(self.search_base) - self.initiate_pagination(request) + + if pagination: + self.initiate_pagination(request) + es_handler = ElasticWrap(self.search_base) response, status_code = es_handler.get(data=self.data) self.response["data"] = SearchProcess(response).process() @@ -74,8 +77,11 @@ class ApiBaseView(APIView): else: self.status_code = 404 - self.pagination_handler.validate(response["hits"]["total"]["value"]) - self.response["paginate"] = self.pagination_handler.pagination + if pagination: + self.pagination_handler.validate( + response["hits"]["total"]["value"] + ) + self.response["paginate"] = self.pagination_handler.pagination class VideoApiView(ApiBaseView): @@ -161,6 +167,30 @@ class VideoCommentView(ApiBaseView): return Response(self.response, status=self.status_code) +class VideoSimilarView(ApiBaseView): + """resolves to /api/video//similar/ + GET: return max 3 videos similar to this + """ + + search_base = "ta_video/_search/" + + def get(self, request, video_id): + """get similar videos""" + self.data = { + "size": 6, + "query": { + "more_like_this": { + "fields": ["tags", "title"], + "like": {"_id": video_id}, + "min_term_freq": 1, + "max_query_terms": 25, + } + }, + } + self.get_document_list(request, pagination=False) + return Response(self.response, status=self.status_code) + + class VideoSponsorView(ApiBaseView): """resolves to /api/video//sponsor/ handle sponsor block integration diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 61a8c2f..3c6a4a5 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -146,11 +146,22 @@ class DownloadPostProcess: if not self.download.config["downloads"]["comment_max"]: return - for video_id in self.download.videos: + total_videos = len(self.download.videos) + for idx, video_id in enumerate(self.download.videos): comment = Comments(video_id, config=self.download.config) - comment.build_json() + comment.build_json(notify=(idx, total_videos)) comment.upload_comments() + key = "message:download" + message = { + "status": key, + "level": "info", + "title": "Download and index comments finished", + "message": f"added comments for {total_videos} videos", + } + + RedisArchivist().set_message(key, message, expire=4) + class VideoDownloader: """ diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 7270563..e3af1d8 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -37,8 +37,7 @@ "type": "text" }, "channel_last_refresh": { - "type": "date", - "format": "epoch_second" + "type": "date" }, "channel_overwrites": { "properties": { @@ -120,8 +119,7 @@ "type": "text" }, "channel_last_refresh": { - "type": "date", - "format": "epoch_second" + "type": "date" }, "channel_overwrites": { "properties": { diff --git a/tubearchivist/home/src/es/index_setup.py b/tubearchivist/home/src/es/index_setup.py index 553e3b9..d7e2d03 100644 --- a/tubearchivist/home/src/es/index_setup.py +++ b/tubearchivist/home/src/es/index_setup.py @@ -100,6 +100,7 @@ class ElasticIndex: def rebuild_index(self): """rebuild with new mapping""" + self.create_blank(for_backup=True) self.reindex("backup") self.delete_index(backup=False) self.create_blank() @@ -126,15 +127,19 @@ class ElasticIndex: _, _ = ElasticWrap(path).delete() - def create_blank(self): + def create_blank(self, for_backup=False): """apply new mapping and settings for blank new index""" + path = f"ta_{self.index_name}" + if for_backup: + path = f"{path}_backup" + data = {} if self.expected_set: data.update({"settings": self.expected_set}) if self.expected_map: data.update({"mappings": {"properties": self.expected_map}}) - _, _ = ElasticWrap(f"ta_{self.index_name}").put(data) + _, _ = ElasticWrap(path).put(data) class BackupCallback: @@ -384,7 +389,9 @@ def backup_all_indexes(reason): index_name = index["index_name"] print(f"backup: export in progress for {index_name}") if not backup_handler.index_exists(index_name): + print(f"skip backup for not yet existing index {index_name}") continue + backup_handler.backup_index(index_name) backup_handler.zip_it() diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index b665ba2..e403fd8 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -116,6 +116,7 @@ class ElasticSnapshot: "repository": self.REPO, "config": { "indices": self.all_indices, + "ignore_unavailable": True, "include_global_state": True, }, "retention": { diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/home/src/frontend/searching.py index 839e196..a082f05 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/home/src/frontend/searching.py @@ -120,7 +120,7 @@ class SearchHandler: if "vid_thumb_url" in hit_keys: youtube_id = hit["source"]["youtube_id"] thumb_path = ThumbManager(youtube_id).vid_thumb_path() - hit["source"]["vid_thumb_url"] = thumb_path + hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}" if "channel_last_refresh" in hit_keys: refreshed = hit["source"]["channel_last_refresh"] diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 7108d43..4b2ecb3 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem): if not self.json_data and fallback: self._video_fallback(fallback) + if not self.json_data: + return + self.get_channel_art() def _video_fallback(self, fallback): diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py index 0ac2436..569f11b 100644 --- a/tubearchivist/home/src/index/comments.py +++ b/tubearchivist/home/src/index/comments.py @@ -10,6 +10,7 @@ from datetime import datetime from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig +from home.src.ta.ta_redis import RedisArchivist class Comments: @@ -23,14 +24,14 @@ class Comments: self.is_activated = False self.comments_format = False - def build_json(self): + def build_json(self, notify=False): """build json document for es""" print(f"{self.youtube_id}: get comments") self.check_config() - if not self.is_activated: return + self._send_notification(notify) comments_raw, channel_id = self.get_yt_comments() self.format_comments(comments_raw) @@ -48,6 +49,23 @@ class Comments: self.is_activated = bool(self.config["downloads"]["comment_max"]) + @staticmethod + def _send_notification(notify): + """send notification for download post process message""" + if not notify: + return + + key = "message:download" + idx, total_videos = notify + message = { + "status": key, + "level": "info", + "title": "Download and index comments", + "message": f"Progress: {idx + 1}/{total_videos}", + } + + RedisArchivist().set_message(key, message) + def build_yt_obs(self): """ get extractor config diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 099c5d4..105f66b 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -56,11 +56,11 @@ class YouTubeItem: "ta_channel": "channel_active", "ta_playlist": "playlist_active", } - update_path = f"{self.index_name}/_update/{self.youtube_id}" + path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true" data = { "script": f"ctx._source.{key_match.get(self.index_name)} = false" } - _, _ = ElasticWrap(update_path).post(data) + _, _ = ElasticWrap(path).post(data) def del_in_es(self): """delete item from elastic search""" diff --git a/tubearchivist/home/src/index/playlist.py b/tubearchivist/home/src/index/playlist.py index 8358aa4..279fbb5 100644 --- a/tubearchivist/home/src/index/playlist.py +++ b/tubearchivist/home/src/index/playlist.py @@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem): if scrape or not self.json_data: self.get_from_youtube() + if not self.youtube_meta: + self.json_data = False + return + self.process_youtube_meta() self.get_entries() self.json_data["playlist_entries"] = self.all_members diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b8b89f0..a372b5b 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -196,6 +196,8 @@ class Reindex: channel.get_from_youtube() if not channel.json_data: channel.deactivate() + channel.get_from_es() + channel.sync_to_videos() return channel.json_data["channel_subscribed"] = subscribed diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 6480f86..a9fd843 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -100,7 +100,7 @@
- video-thumb + video-thumb {% if video.source.player.progress %}
{% else %} diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 96e5b1d..c4e7634 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -72,7 +72,7 @@
- video_thumb + video_thumb {% if show_ignored_only %} ignored {% else %} diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index c85783c..46721ee 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -12,7 +12,7 @@
- video-thumb + video-thumb {% if video.source.player.progress %}
{% else %} @@ -98,7 +98,7 @@
- video-thumb + video-thumb {% if video.source.player.progress %}
{% else %} diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 7d96f20..ca7a6c0 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -102,7 +102,7 @@
- video-thumb + video-thumb {% if video.source.player.progress %}
{% else %} diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index a5fc044..a578423 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -2,8 +2,10 @@ {% block content %} {% load static %} {% load humanize %} -
-
+
+
+
+
@@ -69,15 +71,24 @@

thumbs-down: {{ video.stats.dislike_count|intcomma }}

{% endif %} {% if video.stats.average_rating %} -

Rating: +

{% for star in video.stats.average_rating %} {{ star }} {% endfor %} -

+
{% endif %}
+ {% if video.tags %} +
+
+ {% for tag in video.tags %} + {{ tag }} + {% endfor %} +
+
+ {% endif %} {% if video.description %}

@@ -123,6 +134,12 @@

{% endfor %} {% endif %} +
+

Similar Videos

+
+
+ +
{% if video.comment_count %}

Comments: {{video.comment_count}}

diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 6d5cc23..e69d7bc 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -783,6 +783,18 @@ video:-webkit-full-screen { display: flex; } +.video-tag-box { + display: flex; + flex-wrap: wrap; + justify-content: center; +} + +.video-tag { + padding: 5px 10px; + margin: 5px; + border: 1px solid var(--accent-font-light); +} + .thumb-icon img, .rating-stars img { width: 20px; @@ -826,10 +838,13 @@ video:-webkit-full-screen { .comment-box { padding-bottom: 1rem; + overflow: hidden; } .comments-replies { - padding-left: 3rem; + display: none; + padding-left: 1rem; + border-left: 1px solid var(--accent-font-light); margin-top: 1rem; } diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 7e760bf..aec8566 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) { xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken); xhttp.setRequestHeader('Content-Type', 'application/json'); xhttp.send(JSON.stringify(data)); - return JSON.parse(xhttp.responseText); + if (xhttp.status === 404) { + return false; + } else { + return JSON.parse(xhttp.responseText); + } } // Gets origin URL @@ -951,7 +955,7 @@ function createVideo(video, viewStyle) { // create video item div from template const videoId = video.youtube_id; // const mediaUrl = video.media_url; - const thumbUrl = '/cache/' + video.vid_thumb_url; + // const thumbUrl = '/cache/' + video.vid_thumb_url; const videoTitle = video.title; const videoPublished = video.published; const videoDuration = video.player.duration_str; @@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
- video-thumb + video-thumb
play-icon @@ -1123,12 +1127,18 @@ function writeComments(allComments) { if (rootComment.comment_replies) { let commentReplyBox = document.createElement('div'); commentReplyBox.setAttribute('class', 'comments-replies'); - for (let j = 0; j < rootComment.comment_replies.length; j++) { + commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies'); + let totalReplies = rootComment.comment_replies.length; + if (totalReplies > 0) { + let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies); + commentBox.appendChild(replyButton); + } + for (let j = 0; j < totalReplies; j++) { const commentReply = rootComment.comment_replies[j]; let commentReplyDiv = createCommentBox(commentReply, false); commentReplyBox.appendChild(commentReplyDiv); } - if (rootComment.comment_replies.length > 0) { + if (totalReplies > 0) { commentBox.appendChild(commentReplyBox); } } @@ -1136,6 +1146,27 @@ function writeComments(allComments) { } } +function createReplyButton(replyId, totalReplies) { + let replyButton = document.createElement('button'); + replyButton.innerHTML = `+ ${totalReplies} replies`; + replyButton.setAttribute('data-id', replyId); + replyButton.setAttribute('onclick', 'toggleCommentReplies(this)'); + return replyButton; +} + +function toggleCommentReplies(button) { + let commentReplyId = button.getAttribute('data-id'); + let state = document.getElementById(commentReplyId).style.display; + + if (state === 'none' || state === '') { + document.getElementById(commentReplyId).style.display = 'block'; + button.querySelector('#toggle-icon').innerHTML = '-'; + } else { + document.getElementById(commentReplyId).style.display = 'none'; + button.querySelector('#toggle-icon').innerHTML = '+'; + } +} + function createCommentBox(comment, isRoot) { let commentBox = document.createElement('div'); commentBox.setAttribute('class', 'comment-box'); @@ -1167,7 +1198,7 @@ function createCommentBox(comment, isRoot) { commentMeta.innerHTML = `${comment.comment_time_text}`; if (comment.comment_likecount > 0) { - let numberFormatted = formatNumbers(comment.comment_likecount) + let numberFormatted = formatNumbers(comment.comment_likecount); commentMeta.innerHTML += `${spacer} ${numberFormatted}`; } @@ -1180,6 +1211,35 @@ function createCommentBox(comment, isRoot) { return commentBox; } +function getSimilarVideos(videoId) { + let apiEndpoint = '/api/video/' + videoId + '/similar/'; + let response = apiRequest(apiEndpoint, 'GET'); + if (!response) { + populateEmpty(); + return; + } + let allSimilar = response.data; + if (allSimilar.length > 0) { + populateSimilar(allSimilar); + } +} + +function populateSimilar(allSimilar) { + let similarBox = document.getElementById('similar-videos'); + for (let i = 0; i < allSimilar.length; i++) { + const similarRaw = allSimilar[i]; + let similarDiv = createVideo(similarRaw, 'grid'); + similarBox.appendChild(similarDiv); + } +} + +function populateEmpty() { + let similarBox = document.getElementById('similar-videos'); + let emptyMessage = document.createElement('p'); + emptyMessage.innerText = 'No similar videos found.'; + similarBox.appendChild(emptyMessage); +} + // generic function sendPost(payload) {