video page improvements, #build

Changed:
- Added similar videos
- Added video tag cloud
- Added comment reply toggle
- Added comments progress
- Fixed channel and playlist deactivate

README.md

@@ -65,7 +65,7 @@ There's dedicated user-contributed install steps under [docs/Installation.md](./
 
 For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.  
 
-Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
+Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64.
 
 This project requires docker. Ensure it is installed and running on your system.
 

tubearchivist/api/urls.py

@@ -20,6 +20,7 @@ from api.views import (
     VideoApiView,
     VideoCommentView,
     VideoProgressView,
+    VideoSimilarView,
     VideoSponsorView,
 )
 from django.urls import path
@@ -47,6 +48,11 @@ urlpatterns = [
         VideoCommentView.as_view(),
         name="api-video-comment",
     ),
+    path(
+        "video/<slug:video_id>/similar/",
+        VideoSimilarView.as_view(),
+        name="api-video-similar",
+    ),
     path(
         "video/<slug:video_id>/sponsor/",
         VideoSponsorView.as_view(),

tubearchivist/api/views.py

@@ -62,10 +62,13 @@ class ApiBaseView(APIView):
             }
         )
 
-    def get_document_list(self, request):
+    def get_document_list(self, request, pagination=True):
         """get a list of results"""
         print(self.search_base)
-        self.initiate_pagination(request)
+
+        if pagination:
+            self.initiate_pagination(request)
+
         es_handler = ElasticWrap(self.search_base)
         response, status_code = es_handler.get(data=self.data)
         self.response["data"] = SearchProcess(response).process()
@@ -74,8 +77,11 @@ class ApiBaseView(APIView):
         else:
             self.status_code = 404
 
-        self.pagination_handler.validate(response["hits"]["total"]["value"])
+        if pagination:
-        self.response["paginate"] = self.pagination_handler.pagination
+            self.pagination_handler.validate(
+                response["hits"]["total"]["value"]
+            )
+            self.response["paginate"] = self.pagination_handler.pagination
 
 
 class VideoApiView(ApiBaseView):
@@ -161,6 +167,30 @@ class VideoCommentView(ApiBaseView):
         return Response(self.response, status=self.status_code)
 
 
+class VideoSimilarView(ApiBaseView):
+    """resolves to /api/video/<video-id>/similar/
+    GET: return max 3 videos similar to this
+    """
+
+    search_base = "ta_video/_search/"
+
+    def get(self, request, video_id):
+        """get similar videos"""
+        self.data = {
+            "size": 6,
+            "query": {
+                "more_like_this": {
+                    "fields": ["tags", "title"],
+                    "like": {"_id": video_id},
+                    "min_term_freq": 1,
+                    "max_query_terms": 25,
+                }
+            },
+        }
+        self.get_document_list(request, pagination=False)
+        return Response(self.response, status=self.status_code)
+
+
 class VideoSponsorView(ApiBaseView):
     """resolves to /api/video/<video_id>/sponsor/
     handle sponsor block integration

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -146,11 +146,22 @@ class DownloadPostProcess:
         if not self.download.config["downloads"]["comment_max"]:
             return
 
-        for video_id in self.download.videos:
+        total_videos = len(self.download.videos)
+        for idx, video_id in enumerate(self.download.videos):
             comment = Comments(video_id, config=self.download.config)
-            comment.build_json()
+            comment.build_json(notify=(idx, total_videos))
             comment.upload_comments()
 
+        key = "message:download"
+        message = {
+            "status": key,
+            "level": "info",
+            "title": "Download and index comments finished",
+            "message": f"added comments for {total_videos} videos",
+        }
+
+        RedisArchivist().set_message(key, message, expire=4)
+
 
 class VideoDownloader:
     """

tubearchivist/home/src/es/index_mapping.json

@@ -37,8 +37,7 @@
                     "type": "text"
                 },
                 "channel_last_refresh": {
-                    "type": "date",
+                    "type": "date"
-                    "format": "epoch_second"
                 },
                 "channel_overwrites": {
                     "properties": {
@@ -120,8 +119,7 @@
                             "type": "text"
                         },
                         "channel_last_refresh": {
-                            "type": "date",
+                            "type": "date"
-                            "format": "epoch_second"
                         },
                         "channel_overwrites": {
                             "properties": {

tubearchivist/home/src/es/index_setup.py

@@ -100,6 +100,7 @@ class ElasticIndex:
 
     def rebuild_index(self):
         """rebuild with new mapping"""
+        self.create_blank(for_backup=True)
         self.reindex("backup")
         self.delete_index(backup=False)
         self.create_blank()
@@ -126,15 +127,19 @@ class ElasticIndex:
 
         _, _ = ElasticWrap(path).delete()
 
-    def create_blank(self):
+    def create_blank(self, for_backup=False):
         """apply new mapping and settings for blank new index"""
+        path = f"ta_{self.index_name}"
+        if for_backup:
+            path = f"{path}_backup"
+
         data = {}
         if self.expected_set:
             data.update({"settings": self.expected_set})
         if self.expected_map:
             data.update({"mappings": {"properties": self.expected_map}})
 
-        _, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
+        _, _ = ElasticWrap(path).put(data)
 
 
 class BackupCallback:
@@ -384,7 +389,9 @@ def backup_all_indexes(reason):
         index_name = index["index_name"]
         print(f"backup: export in progress for {index_name}")
         if not backup_handler.index_exists(index_name):
+            print(f"skip backup for not yet existing index {index_name}")
             continue
+
         backup_handler.backup_index(index_name)
 
     backup_handler.zip_it()

tubearchivist/home/src/es/snapshot.py

@@ -116,6 +116,7 @@ class ElasticSnapshot:
             "repository": self.REPO,
             "config": {
                 "indices": self.all_indices,
+                "ignore_unavailable": True,
                 "include_global_state": True,
             },
             "retention": {

tubearchivist/home/src/frontend/searching.py

@@ -120,7 +120,7 @@ class SearchHandler:
         if "vid_thumb_url" in hit_keys:
             youtube_id = hit["source"]["youtube_id"]
             thumb_path = ThumbManager(youtube_id).vid_thumb_path()
-            hit["source"]["vid_thumb_url"] = thumb_path
+            hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
 
         if "channel_last_refresh" in hit_keys:
             refreshed = hit["source"]["channel_last_refresh"]

tubearchivist/home/src/index/channel.py

@@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem):
         if not self.json_data and fallback:
             self._video_fallback(fallback)
 
+        if not self.json_data:
+            return
+
         self.get_channel_art()
 
     def _video_fallback(self, fallback):

tubearchivist/home/src/index/comments.py

@@ -10,6 +10,7 @@ from datetime import datetime
 from home.src.download.yt_dlp_base import YtWrap
 from home.src.es.connect import ElasticWrap
 from home.src.ta.config import AppConfig
+from home.src.ta.ta_redis import RedisArchivist
 
 
 class Comments:
@@ -23,14 +24,14 @@ class Comments:
         self.is_activated = False
         self.comments_format = False
 
-    def build_json(self):
+    def build_json(self, notify=False):
         """build json document for es"""
         print(f"{self.youtube_id}: get comments")
         self.check_config()
-
         if not self.is_activated:
             return
 
+        self._send_notification(notify)
         comments_raw, channel_id = self.get_yt_comments()
         self.format_comments(comments_raw)
 
@@ -48,6 +49,23 @@ class Comments:
 
         self.is_activated = bool(self.config["downloads"]["comment_max"])
 
+    @staticmethod
+    def _send_notification(notify):
+        """send notification for download post process message"""
+        if not notify:
+            return
+
+        key = "message:download"
+        idx, total_videos = notify
+        message = {
+            "status": key,
+            "level": "info",
+            "title": "Download and index comments",
+            "message": f"Progress: {idx + 1}/{total_videos}",
+        }
+
+        RedisArchivist().set_message(key, message)
+
     def build_yt_obs(self):
         """
         get extractor config

tubearchivist/home/src/index/generic.py

@@ -56,11 +56,11 @@ class YouTubeItem:
             "ta_channel": "channel_active",
             "ta_playlist": "playlist_active",
         }
-        update_path = f"{self.index_name}/_update/{self.youtube_id}"
+        path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
         data = {
             "script": f"ctx._source.{key_match.get(self.index_name)} = false"
         }
-        _, _ = ElasticWrap(update_path).post(data)
+        _, _ = ElasticWrap(path).post(data)
 
     def del_in_es(self):
         """delete item from elastic search"""

tubearchivist/home/src/index/playlist.py

@@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem):
 
         if scrape or not self.json_data:
             self.get_from_youtube()
+            if not self.youtube_meta:
+                self.json_data = False
+                return
+
             self.process_youtube_meta()
             self.get_entries()
             self.json_data["playlist_entries"] = self.all_members

tubearchivist/home/src/index/reindex.py

@@ -196,6 +196,8 @@ class Reindex:
         channel.get_from_youtube()
         if not channel.json_data:
             channel.deactivate()
+            channel.get_from_es()
+            channel.sync_to_videos()
             return
 
         channel.json_data["channel_subscribed"] = subscribed

tubearchivist/home/templates/home/channel_id.html

@@ -100,7 +100,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}

tubearchivist/home/templates/home/downloads.html

@@ -72,7 +72,7 @@
                 <div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
                     <div class="video-thumb-wrap {{ view_style }}">
                         <div class="video-thumb">
-                            <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
+                            <img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
                             {% if show_ignored_only %}
                                 <span>ignored</span>
                             {% else %}

tubearchivist/home/templates/home/home.html

@@ -12,7 +12,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}
@@ -98,7 +98,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}

tubearchivist/home/templates/home/playlist_id.html

@@ -102,7 +102,7 @@
                     <a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
                         <div class="video-thumb-wrap {{ view_style }}">
                             <div class="video-thumb">
-                                <img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
+                                <img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
                                 {% if video.source.player.progress %}
                                     <div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
                                 {% else %}

tubearchivist/home/templates/home/video.html

@@ -2,8 +2,10 @@
 {% block content %}
 {% load static %}
 {% load humanize %}
-<div class="video-main">
+<div id="player" class="player-wrapper">
-    <div class="video-modal"><span class="video-modal-text"></span></div>
+    <div class="video-main">
+        <div class="video-modal"><span class="video-modal-text"></span></div>
+    </div>
 </div>
 <div class="notifications" id="notifications"></div>
 <div class="sponsorblock" id="sponsorblock">
@@ -69,15 +71,24 @@
                     <p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
                 {% endif %}
                 {% if video.stats.average_rating %}
-                    <p class="rating-stars">Rating:
+                    <div class="rating-stars">
                         {% for star in video.stats.average_rating %}
                             <img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
                         {% endfor %}
-                    </p>
+                    </div>
                 {% endif %}
             </div>
         </div>
     </div>
+    {% if video.tags %}
+        <div class="description-box">
+            <div class="video-tag-box">
+                {% for tag in video.tags %}
+                    <span class="video-tag">{{ tag }}</span>
+                {% endfor %}
+            </div>
+        </div>
+    {% endif %}
     {% if video.description %}
         <div class="description-box">
             <p id="text-expand" class="description-text">
@@ -123,6 +134,12 @@
             </div>
         {% endfor %}
     {% endif %}
+    <div class="description-box">
+        <h3>Similar Videos</h3>
+        <div class="video-list grid grid-3" id="similar-videos">
+        </div>
+        <script>getSimilarVideos('{{ video.youtube_id }}')</script>
+    </div>
     {% if video.comment_count %}
         <div class="comments-section">
             <h3>Comments: {{video.comment_count}}</h3>

tubearchivist/static/css/style.css

@@ -783,6 +783,18 @@ video:-webkit-full-screen {
     display: flex;
 }
 
+.video-tag-box {
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+}
+
+.video-tag {
+    padding: 5px 10px;
+    margin: 5px;
+    border: 1px solid var(--accent-font-light);
+}
+
 .thumb-icon img,
 .rating-stars img {
     width: 20px;
@@ -826,10 +838,13 @@ video:-webkit-full-screen {
 
 .comment-box {
     padding-bottom: 1rem;
+    overflow: hidden;
 }
 
 .comments-replies {
-    padding-left: 3rem;
+    display: none;
+    padding-left: 1rem;
+    border-left: 1px solid var(--accent-font-light);
     margin-top: 1rem;
 }
 

tubearchivist/static/script.js

@@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) {
   xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken);
   xhttp.setRequestHeader('Content-Type', 'application/json');
   xhttp.send(JSON.stringify(data));
-  return JSON.parse(xhttp.responseText);
+  if (xhttp.status === 404) {
+    return false;
+  } else {
+    return JSON.parse(xhttp.responseText);
+  }
 }
 
 // Gets origin URL
@@ -951,7 +955,7 @@ function createVideo(video, viewStyle) {
   // create video item div from template
   const videoId = video.youtube_id;
   // const mediaUrl = video.media_url;
-  const thumbUrl = '/cache/' + video.vid_thumb_url;
+  // const thumbUrl = '/cache/' + video.vid_thumb_url;
   const videoTitle = video.title;
   const videoPublished = video.published;
   const videoDuration = video.player.duration_str;
@@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
     <a href="#player" data-id="${videoId}" onclick="createPlayer(this)">
         <div class="video-thumb-wrap ${viewStyle}">
             <div class="video-thumb">
-                <img src="${thumbUrl}" alt="video-thumb">
+                <img src="${video.vid_thumb_url}" alt="video-thumb">
             </div>
             <div class="video-play">
                 <img src="/static/img/icon-play.svg" alt="play-icon">
@@ -1123,12 +1127,18 @@ function writeComments(allComments) {
     if (rootComment.comment_replies) {
       let commentReplyBox = document.createElement('div');
       commentReplyBox.setAttribute('class', 'comments-replies');
-      for (let j = 0; j < rootComment.comment_replies.length; j++) {
+      commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies');
+      let totalReplies = rootComment.comment_replies.length;
+      if (totalReplies > 0) {
+        let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies);
+        commentBox.appendChild(replyButton);
+      }
+      for (let j = 0; j < totalReplies; j++) {
         const commentReply = rootComment.comment_replies[j];
         let commentReplyDiv = createCommentBox(commentReply, false);
         commentReplyBox.appendChild(commentReplyDiv);
       }
-      if (rootComment.comment_replies.length > 0) {
+      if (totalReplies > 0) {
         commentBox.appendChild(commentReplyBox);
       }
     }
@@ -1136,6 +1146,27 @@ function writeComments(allComments) {
   }
 }
 
+function createReplyButton(replyId, totalReplies) {
+  let replyButton = document.createElement('button');
+  replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
+  replyButton.setAttribute('data-id', replyId);
+  replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
+  return replyButton;
+}
+
+function toggleCommentReplies(button) {
+  let commentReplyId = button.getAttribute('data-id');
+  let state = document.getElementById(commentReplyId).style.display;
+
+  if (state === 'none' || state === '') {
+    document.getElementById(commentReplyId).style.display = 'block';
+    button.querySelector('#toggle-icon').innerHTML = '-';
+  } else {
+    document.getElementById(commentReplyId).style.display = 'none';
+    button.querySelector('#toggle-icon').innerHTML = '+';
+  }
+}
+
 function createCommentBox(comment, isRoot) {
   let commentBox = document.createElement('div');
   commentBox.setAttribute('class', 'comment-box');
@@ -1167,7 +1198,7 @@ function createCommentBox(comment, isRoot) {
   commentMeta.innerHTML = `<span>${comment.comment_time_text}</span>`;
 
   if (comment.comment_likecount > 0) {
-    let numberFormatted = formatNumbers(comment.comment_likecount)
+    let numberFormatted = formatNumbers(comment.comment_likecount);
     commentMeta.innerHTML += `${spacer}<span class="thumb-icon"><img src="/static/img/icon-thumb.svg"> ${numberFormatted}</span>`;
   }
 
@@ -1180,6 +1211,35 @@ function createCommentBox(comment, isRoot) {
   return commentBox;
 }
 
+function getSimilarVideos(videoId) {
+  let apiEndpoint = '/api/video/' + videoId + '/similar/';
+  let response = apiRequest(apiEndpoint, 'GET');
+  if (!response) {
+    populateEmpty();
+    return;
+  }
+  let allSimilar = response.data;
+  if (allSimilar.length > 0) {
+    populateSimilar(allSimilar);
+  }
+}
+
+function populateSimilar(allSimilar) {
+  let similarBox = document.getElementById('similar-videos');
+  for (let i = 0; i < allSimilar.length; i++) {
+    const similarRaw = allSimilar[i];
+    let similarDiv = createVideo(similarRaw, 'grid');
+    similarBox.appendChild(similarDiv);
+  }
+}
+
+function populateEmpty() {
+  let similarBox = document.getElementById('similar-videos');
+  let emptyMessage = document.createElement('p');
+  emptyMessage.innerText = 'No similar videos found.';
+  similarBox.appendChild(emptyMessage);
+}
+
 // generic
 
 function sendPost(payload) {