video page improvements, #build

Changed:
- Added similar videos
- Added video tag cloud
- Added comment reply toggle
- Added comments progress
- Fixed channel and playlist deactivate
pull/370/head
simon 2 weeks ago
commit 24d66e33a7
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4

@ -65,7 +65,7 @@ There's dedicated user-contributed install steps under [docs/Installation.md](./
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64.
This project requires docker. Ensure it is installed and running on your system.

@ -20,6 +20,7 @@ from api.views import (
VideoApiView,
VideoCommentView,
VideoProgressView,
VideoSimilarView,
VideoSponsorView,
)
from django.urls import path
@ -47,6 +48,11 @@ urlpatterns = [
VideoCommentView.as_view(),
name="api-video-comment",
),
path(
"video/<slug:video_id>/similar/",
VideoSimilarView.as_view(),
name="api-video-similar",
),
path(
"video/<slug:video_id>/sponsor/",
VideoSponsorView.as_view(),

@ -62,10 +62,13 @@ class ApiBaseView(APIView):
}
)
def get_document_list(self, request):
def get_document_list(self, request, pagination=True):
"""get a list of results"""
print(self.search_base)
self.initiate_pagination(request)
if pagination:
self.initiate_pagination(request)
es_handler = ElasticWrap(self.search_base)
response, status_code = es_handler.get(data=self.data)
self.response["data"] = SearchProcess(response).process()
@ -74,8 +77,11 @@ class ApiBaseView(APIView):
else:
self.status_code = 404
self.pagination_handler.validate(response["hits"]["total"]["value"])
self.response["paginate"] = self.pagination_handler.pagination
if pagination:
self.pagination_handler.validate(
response["hits"]["total"]["value"]
)
self.response["paginate"] = self.pagination_handler.pagination
class VideoApiView(ApiBaseView):
@ -161,6 +167,30 @@ class VideoCommentView(ApiBaseView):
return Response(self.response, status=self.status_code)
class VideoSimilarView(ApiBaseView):
"""resolves to /api/video/<video-id>/similar/
GET: return max 3 videos similar to this
"""
search_base = "ta_video/_search/"
def get(self, request, video_id):
"""get similar videos"""
self.data = {
"size": 6,
"query": {
"more_like_this": {
"fields": ["tags", "title"],
"like": {"_id": video_id},
"min_term_freq": 1,
"max_query_terms": 25,
}
},
}
self.get_document_list(request, pagination=False)
return Response(self.response, status=self.status_code)
class VideoSponsorView(ApiBaseView):
"""resolves to /api/video/<video_id>/sponsor/
handle sponsor block integration

@ -146,11 +146,22 @@ class DownloadPostProcess:
if not self.download.config["downloads"]["comment_max"]:
return
for video_id in self.download.videos:
total_videos = len(self.download.videos)
for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config)
comment.build_json()
comment.build_json(notify=(idx, total_videos))
comment.upload_comments()
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)
class VideoDownloader:
"""

@ -37,8 +37,7 @@
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
"type": "date"
},
"channel_overwrites": {
"properties": {
@ -120,8 +119,7 @@
"type": "text"
},
"channel_last_refresh": {
"type": "date",
"format": "epoch_second"
"type": "date"
},
"channel_overwrites": {
"properties": {

@ -100,6 +100,7 @@ class ElasticIndex:
def rebuild_index(self):
"""rebuild with new mapping"""
self.create_blank(for_backup=True)
self.reindex("backup")
self.delete_index(backup=False)
self.create_blank()
@ -126,15 +127,19 @@ class ElasticIndex:
_, _ = ElasticWrap(path).delete()
def create_blank(self):
def create_blank(self, for_backup=False):
"""apply new mapping and settings for blank new index"""
path = f"ta_{self.index_name}"
if for_backup:
path = f"{path}_backup"
data = {}
if self.expected_set:
data.update({"settings": self.expected_set})
if self.expected_map:
data.update({"mappings": {"properties": self.expected_map}})
_, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
_, _ = ElasticWrap(path).put(data)
class BackupCallback:
@ -384,7 +389,9 @@ def backup_all_indexes(reason):
index_name = index["index_name"]
print(f"backup: export in progress for {index_name}")
if not backup_handler.index_exists(index_name):
print(f"skip backup for not yet existing index {index_name}")
continue
backup_handler.backup_index(index_name)
backup_handler.zip_it()

@ -116,6 +116,7 @@ class ElasticSnapshot:
"repository": self.REPO,
"config": {
"indices": self.all_indices,
"ignore_unavailable": True,
"include_global_state": True,
},
"retention": {

@ -120,7 +120,7 @@ class SearchHandler:
if "vid_thumb_url" in hit_keys:
youtube_id = hit["source"]["youtube_id"]
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
hit["source"]["vid_thumb_url"] = thumb_path
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
if "channel_last_refresh" in hit_keys:
refreshed = hit["source"]["channel_last_refresh"]

@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem):
if not self.json_data and fallback:
self._video_fallback(fallback)
if not self.json_data:
return
self.get_channel_art()
def _video_fallback(self, fallback):

@ -10,6 +10,7 @@ from datetime import datetime
from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap
from home.src.ta.config import AppConfig
from home.src.ta.ta_redis import RedisArchivist
class Comments:
@ -23,14 +24,14 @@ class Comments:
self.is_activated = False
self.comments_format = False
def build_json(self):
def build_json(self, notify=False):
"""build json document for es"""
print(f"{self.youtube_id}: get comments")
self.check_config()
if not self.is_activated:
return
self._send_notification(notify)
comments_raw, channel_id = self.get_yt_comments()
self.format_comments(comments_raw)
@ -48,6 +49,23 @@ class Comments:
self.is_activated = bool(self.config["downloads"]["comment_max"])
@staticmethod
def _send_notification(notify):
"""send notification for download post process message"""
if not notify:
return
key = "message:download"
idx, total_videos = notify
message = {
"status": key,
"level": "info",
"title": "Download and index comments",
"message": f"Progress: {idx + 1}/{total_videos}",
}
RedisArchivist().set_message(key, message)
def build_yt_obs(self):
"""
get extractor config

@ -56,11 +56,11 @@ class YouTubeItem:
"ta_channel": "channel_active",
"ta_playlist": "playlist_active",
}
update_path = f"{self.index_name}/_update/{self.youtube_id}"
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
data = {
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
}
_, _ = ElasticWrap(update_path).post(data)
_, _ = ElasticWrap(path).post(data)
def del_in_es(self):
"""delete item from elastic search"""

@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem):
if scrape or not self.json_data:
self.get_from_youtube()
if not self.youtube_meta:
self.json_data = False
return
self.process_youtube_meta()
self.get_entries()
self.json_data["playlist_entries"] = self.all_members

@ -196,6 +196,8 @@ class Reindex:
channel.get_from_youtube()
if not channel.json_data:
channel.deactivate()
channel.get_from_es()
channel.sync_to_videos()
return
channel.json_data["channel_subscribed"] = subscribed

@ -100,7 +100,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

@ -72,7 +72,7 @@
<div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
{% if show_ignored_only %}
<span>ignored</span>
{% else %}

@ -12,7 +12,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}
@ -98,7 +98,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

@ -102,7 +102,7 @@
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
<div class="video-thumb-wrap {{ view_style }}">
<div class="video-thumb">
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
{% if video.source.player.progress %}
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
{% else %}

@ -2,8 +2,10 @@
{% block content %}
{% load static %}
{% load humanize %}
<div class="video-main">
<div class="video-modal"><span class="video-modal-text"></span></div>
<div id="player" class="player-wrapper">
<div class="video-main">
<div class="video-modal"><span class="video-modal-text"></span></div>
</div>
</div>
<div class="notifications" id="notifications"></div>
<div class="sponsorblock" id="sponsorblock">
@ -69,15 +71,24 @@
<p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
{% endif %}
{% if video.stats.average_rating %}
<p class="rating-stars">Rating:
<div class="rating-stars">
{% for star in video.stats.average_rating %}
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
{% endfor %}
</p>
</div>
{% endif %}
</div>
</div>
</div>
{% if video.tags %}
<div class="description-box">
<div class="video-tag-box">
{% for tag in video.tags %}
<span class="video-tag">{{ tag }}</span>
{% endfor %}
</div>
</div>
{% endif %}
{% if video.description %}
<div class="description-box">
<p id="text-expand" class="description-text">
@ -123,6 +134,12 @@
</div>
{% endfor %}
{% endif %}
<div class="description-box">
<h3>Similar Videos</h3>
<div class="video-list grid grid-3" id="similar-videos">
</div>
<script>getSimilarVideos('{{ video.youtube_id }}')</script>
</div>
{% if video.comment_count %}
<div class="comments-section">
<h3>Comments: {{video.comment_count}}</h3>

@ -783,6 +783,18 @@ video:-webkit-full-screen {
display: flex;
}
.video-tag-box {
display: flex;
flex-wrap: wrap;
justify-content: center;
}
.video-tag {
padding: 5px 10px;
margin: 5px;
border: 1px solid var(--accent-font-light);
}
.thumb-icon img,
.rating-stars img {
width: 20px;
@ -826,10 +838,13 @@ video:-webkit-full-screen {
.comment-box {
padding-bottom: 1rem;
overflow: hidden;
}
.comments-replies {
padding-left: 3rem;
display: none;
padding-left: 1rem;
border-left: 1px solid var(--accent-font-light);
margin-top: 1rem;
}

@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) {
xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken);
xhttp.setRequestHeader('Content-Type', 'application/json');
xhttp.send(JSON.stringify(data));
return JSON.parse(xhttp.responseText);
if (xhttp.status === 404) {
return false;
} else {
return JSON.parse(xhttp.responseText);
}
}
// Gets origin URL
@ -951,7 +955,7 @@ function createVideo(video, viewStyle) {
// create video item div from template
const videoId = video.youtube_id;
// const mediaUrl = video.media_url;
const thumbUrl = '/cache/' + video.vid_thumb_url;
// const thumbUrl = '/cache/' + video.vid_thumb_url;
const videoTitle = video.title;
const videoPublished = video.published;
const videoDuration = video.player.duration_str;
@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
<a href="#player" data-id="${videoId}" onclick="createPlayer(this)">
<div class="video-thumb-wrap ${viewStyle}">
<div class="video-thumb">
<img src="${thumbUrl}" alt="video-thumb">
<img src="${video.vid_thumb_url}" alt="video-thumb">
</div>
<div class="video-play">
<img src="/static/img/icon-play.svg" alt="play-icon">
@ -1123,12 +1127,18 @@ function writeComments(allComments) {
if (rootComment.comment_replies) {
let commentReplyBox = document.createElement('div');
commentReplyBox.setAttribute('class', 'comments-replies');
for (let j = 0; j < rootComment.comment_replies.length; j++) {
commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies');
let totalReplies = rootComment.comment_replies.length;
if (totalReplies > 0) {
let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies);
commentBox.appendChild(replyButton);
}
for (let j = 0; j < totalReplies; j++) {
const commentReply = rootComment.comment_replies[j];
let commentReplyDiv = createCommentBox(commentReply, false);
commentReplyBox.appendChild(commentReplyDiv);
}
if (rootComment.comment_replies.length > 0) {
if (totalReplies > 0) {
commentBox.appendChild(commentReplyBox);
}
}
@ -1136,6 +1146,27 @@ function writeComments(allComments) {
}
}
function createReplyButton(replyId, totalReplies) {
let replyButton = document.createElement('button');
replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
replyButton.setAttribute('data-id', replyId);
replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
return replyButton;
}
function toggleCommentReplies(button) {
let commentReplyId = button.getAttribute('data-id');
let state = document.getElementById(commentReplyId).style.display;
if (state === 'none' || state === '') {
document.getElementById(commentReplyId).style.display = 'block';
button.querySelector('#toggle-icon').innerHTML = '-';
} else {
document.getElementById(commentReplyId).style.display = 'none';
button.querySelector('#toggle-icon').innerHTML = '+';
}
}
function createCommentBox(comment, isRoot) {
let commentBox = document.createElement('div');
commentBox.setAttribute('class', 'comment-box');
@ -1167,7 +1198,7 @@ function createCommentBox(comment, isRoot) {
commentMeta.innerHTML = `<span>${comment.comment_time_text}</span>`;
if (comment.comment_likecount > 0) {
let numberFormatted = formatNumbers(comment.comment_likecount)
let numberFormatted = formatNumbers(comment.comment_likecount);
commentMeta.innerHTML += `${spacer}<span class="thumb-icon"><img src="/static/img/icon-thumb.svg"> ${numberFormatted}</span>`;
}
@ -1180,6 +1211,35 @@ function createCommentBox(comment, isRoot) {
return commentBox;
}
function getSimilarVideos(videoId) {
let apiEndpoint = '/api/video/' + videoId + '/similar/';
let response = apiRequest(apiEndpoint, 'GET');
if (!response) {
populateEmpty();
return;
}
let allSimilar = response.data;
if (allSimilar.length > 0) {
populateSimilar(allSimilar);
}
}
function populateSimilar(allSimilar) {
let similarBox = document.getElementById('similar-videos');
for (let i = 0; i < allSimilar.length; i++) {
const similarRaw = allSimilar[i];
let similarDiv = createVideo(similarRaw, 'grid');
similarBox.appendChild(similarDiv);
}
}
function populateEmpty() {
let similarBox = document.getElementById('similar-videos');
let emptyMessage = document.createElement('p');
emptyMessage.innerText = 'No similar videos found.';
similarBox.appendChild(emptyMessage);
}
// generic
function sendPost(payload) {

Loading…
Cancel
Save