video page improvements, #build
Changed: - Added similar videos - Added video tag cloud - Added comment reply toggle - Added comments progress - Fixed channel and playlist deactivate
This commit is contained in:
commit
24d66e33a7
|
@ -65,7 +65,7 @@ There's dedicated user-contributed install steps under [docs/Installation.md](./
|
|||
|
||||
For minimal system requirements, the Tube Archivist stack needs around 2GB of available memory for a small testing setup and around 4GB of available memory for a mid to large sized installation.
|
||||
|
||||
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson` (an unofficial rebuild for arm64) instead of [the official one](https://github.com/tubearchivist/tubearchivist/blob/4af12aee15620e330adf3624c984c3acf6d0ac8b/docker-compose.yml#L27).
|
||||
Note for arm64 hosts: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, but you can use the image `bbilly1/rejson`, an unofficial rebuild for arm64.
|
||||
|
||||
This project requires docker. Ensure it is installed and running on your system.
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@ from api.views import (
|
|||
VideoApiView,
|
||||
VideoCommentView,
|
||||
VideoProgressView,
|
||||
VideoSimilarView,
|
||||
VideoSponsorView,
|
||||
)
|
||||
from django.urls import path
|
||||
|
@ -47,6 +48,11 @@ urlpatterns = [
|
|||
VideoCommentView.as_view(),
|
||||
name="api-video-comment",
|
||||
),
|
||||
path(
|
||||
"video/<slug:video_id>/similar/",
|
||||
VideoSimilarView.as_view(),
|
||||
name="api-video-similar",
|
||||
),
|
||||
path(
|
||||
"video/<slug:video_id>/sponsor/",
|
||||
VideoSponsorView.as_view(),
|
||||
|
|
|
@ -62,10 +62,13 @@ class ApiBaseView(APIView):
|
|||
}
|
||||
)
|
||||
|
||||
def get_document_list(self, request):
|
||||
def get_document_list(self, request, pagination=True):
|
||||
"""get a list of results"""
|
||||
print(self.search_base)
|
||||
|
||||
if pagination:
|
||||
self.initiate_pagination(request)
|
||||
|
||||
es_handler = ElasticWrap(self.search_base)
|
||||
response, status_code = es_handler.get(data=self.data)
|
||||
self.response["data"] = SearchProcess(response).process()
|
||||
|
@ -74,7 +77,10 @@ class ApiBaseView(APIView):
|
|||
else:
|
||||
self.status_code = 404
|
||||
|
||||
self.pagination_handler.validate(response["hits"]["total"]["value"])
|
||||
if pagination:
|
||||
self.pagination_handler.validate(
|
||||
response["hits"]["total"]["value"]
|
||||
)
|
||||
self.response["paginate"] = self.pagination_handler.pagination
|
||||
|
||||
|
||||
|
@ -161,6 +167,30 @@ class VideoCommentView(ApiBaseView):
|
|||
return Response(self.response, status=self.status_code)
|
||||
|
||||
|
||||
class VideoSimilarView(ApiBaseView):
|
||||
"""resolves to /api/video/<video-id>/similar/
|
||||
GET: return max 3 videos similar to this
|
||||
"""
|
||||
|
||||
search_base = "ta_video/_search/"
|
||||
|
||||
def get(self, request, video_id):
|
||||
"""get similar videos"""
|
||||
self.data = {
|
||||
"size": 6,
|
||||
"query": {
|
||||
"more_like_this": {
|
||||
"fields": ["tags", "title"],
|
||||
"like": {"_id": video_id},
|
||||
"min_term_freq": 1,
|
||||
"max_query_terms": 25,
|
||||
}
|
||||
},
|
||||
}
|
||||
self.get_document_list(request, pagination=False)
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
|
||||
class VideoSponsorView(ApiBaseView):
|
||||
"""resolves to /api/video/<video_id>/sponsor/
|
||||
handle sponsor block integration
|
||||
|
|
|
@ -146,11 +146,22 @@ class DownloadPostProcess:
|
|||
if not self.download.config["downloads"]["comment_max"]:
|
||||
return
|
||||
|
||||
for video_id in self.download.videos:
|
||||
total_videos = len(self.download.videos)
|
||||
for idx, video_id in enumerate(self.download.videos):
|
||||
comment = Comments(video_id, config=self.download.config)
|
||||
comment.build_json()
|
||||
comment.build_json(notify=(idx, total_videos))
|
||||
comment.upload_comments()
|
||||
|
||||
key = "message:download"
|
||||
message = {
|
||||
"status": key,
|
||||
"level": "info",
|
||||
"title": "Download and index comments finished",
|
||||
"message": f"added comments for {total_videos} videos",
|
||||
}
|
||||
|
||||
RedisArchivist().set_message(key, message, expire=4)
|
||||
|
||||
|
||||
class VideoDownloader:
|
||||
"""
|
||||
|
|
|
@ -37,8 +37,7 @@
|
|||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"type": "date"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
|
@ -120,8 +119,7 @@
|
|||
"type": "text"
|
||||
},
|
||||
"channel_last_refresh": {
|
||||
"type": "date",
|
||||
"format": "epoch_second"
|
||||
"type": "date"
|
||||
},
|
||||
"channel_overwrites": {
|
||||
"properties": {
|
||||
|
|
|
@ -100,6 +100,7 @@ class ElasticIndex:
|
|||
|
||||
def rebuild_index(self):
|
||||
"""rebuild with new mapping"""
|
||||
self.create_blank(for_backup=True)
|
||||
self.reindex("backup")
|
||||
self.delete_index(backup=False)
|
||||
self.create_blank()
|
||||
|
@ -126,15 +127,19 @@ class ElasticIndex:
|
|||
|
||||
_, _ = ElasticWrap(path).delete()
|
||||
|
||||
def create_blank(self):
|
||||
def create_blank(self, for_backup=False):
|
||||
"""apply new mapping and settings for blank new index"""
|
||||
path = f"ta_{self.index_name}"
|
||||
if for_backup:
|
||||
path = f"{path}_backup"
|
||||
|
||||
data = {}
|
||||
if self.expected_set:
|
||||
data.update({"settings": self.expected_set})
|
||||
if self.expected_map:
|
||||
data.update({"mappings": {"properties": self.expected_map}})
|
||||
|
||||
_, _ = ElasticWrap(f"ta_{self.index_name}").put(data)
|
||||
_, _ = ElasticWrap(path).put(data)
|
||||
|
||||
|
||||
class BackupCallback:
|
||||
|
@ -384,7 +389,9 @@ def backup_all_indexes(reason):
|
|||
index_name = index["index_name"]
|
||||
print(f"backup: export in progress for {index_name}")
|
||||
if not backup_handler.index_exists(index_name):
|
||||
print(f"skip backup for not yet existing index {index_name}")
|
||||
continue
|
||||
|
||||
backup_handler.backup_index(index_name)
|
||||
|
||||
backup_handler.zip_it()
|
||||
|
|
|
@ -116,6 +116,7 @@ class ElasticSnapshot:
|
|||
"repository": self.REPO,
|
||||
"config": {
|
||||
"indices": self.all_indices,
|
||||
"ignore_unavailable": True,
|
||||
"include_global_state": True,
|
||||
},
|
||||
"retention": {
|
||||
|
|
|
@ -120,7 +120,7 @@ class SearchHandler:
|
|||
if "vid_thumb_url" in hit_keys:
|
||||
youtube_id = hit["source"]["youtube_id"]
|
||||
thumb_path = ThumbManager(youtube_id).vid_thumb_path()
|
||||
hit["source"]["vid_thumb_url"] = thumb_path
|
||||
hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}"
|
||||
|
||||
if "channel_last_refresh" in hit_keys:
|
||||
refreshed = hit["source"]["channel_last_refresh"]
|
||||
|
|
|
@ -193,6 +193,9 @@ class YoutubeChannel(YouTubeItem):
|
|||
if not self.json_data and fallback:
|
||||
self._video_fallback(fallback)
|
||||
|
||||
if not self.json_data:
|
||||
return
|
||||
|
||||
self.get_channel_art()
|
||||
|
||||
def _video_fallback(self, fallback):
|
||||
|
|
|
@ -10,6 +10,7 @@ from datetime import datetime
|
|||
from home.src.download.yt_dlp_base import YtWrap
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.ta.config import AppConfig
|
||||
from home.src.ta.ta_redis import RedisArchivist
|
||||
|
||||
|
||||
class Comments:
|
||||
|
@ -23,14 +24,14 @@ class Comments:
|
|||
self.is_activated = False
|
||||
self.comments_format = False
|
||||
|
||||
def build_json(self):
|
||||
def build_json(self, notify=False):
|
||||
"""build json document for es"""
|
||||
print(f"{self.youtube_id}: get comments")
|
||||
self.check_config()
|
||||
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
self._send_notification(notify)
|
||||
comments_raw, channel_id = self.get_yt_comments()
|
||||
self.format_comments(comments_raw)
|
||||
|
||||
|
@ -48,6 +49,23 @@ class Comments:
|
|||
|
||||
self.is_activated = bool(self.config["downloads"]["comment_max"])
|
||||
|
||||
@staticmethod
|
||||
def _send_notification(notify):
|
||||
"""send notification for download post process message"""
|
||||
if not notify:
|
||||
return
|
||||
|
||||
key = "message:download"
|
||||
idx, total_videos = notify
|
||||
message = {
|
||||
"status": key,
|
||||
"level": "info",
|
||||
"title": "Download and index comments",
|
||||
"message": f"Progress: {idx + 1}/{total_videos}",
|
||||
}
|
||||
|
||||
RedisArchivist().set_message(key, message)
|
||||
|
||||
def build_yt_obs(self):
|
||||
"""
|
||||
get extractor config
|
||||
|
|
|
@ -56,11 +56,11 @@ class YouTubeItem:
|
|||
"ta_channel": "channel_active",
|
||||
"ta_playlist": "playlist_active",
|
||||
}
|
||||
update_path = f"{self.index_name}/_update/{self.youtube_id}"
|
||||
path = f"{self.index_name}/_update/{self.youtube_id}?refresh=true"
|
||||
data = {
|
||||
"script": f"ctx._source.{key_match.get(self.index_name)} = false"
|
||||
}
|
||||
_, _ = ElasticWrap(update_path).post(data)
|
||||
_, _ = ElasticWrap(path).post(data)
|
||||
|
||||
def del_in_es(self):
|
||||
"""delete item from elastic search"""
|
||||
|
|
|
@ -41,6 +41,10 @@ class YoutubePlaylist(YouTubeItem):
|
|||
|
||||
if scrape or not self.json_data:
|
||||
self.get_from_youtube()
|
||||
if not self.youtube_meta:
|
||||
self.json_data = False
|
||||
return
|
||||
|
||||
self.process_youtube_meta()
|
||||
self.get_entries()
|
||||
self.json_data["playlist_entries"] = self.all_members
|
||||
|
|
|
@ -196,6 +196,8 @@ class Reindex:
|
|||
channel.get_from_youtube()
|
||||
if not channel.json_data:
|
||||
channel.deactivate()
|
||||
channel.get_from_es()
|
||||
channel.sync_to_videos()
|
||||
return
|
||||
|
||||
channel.json_data["channel_subscribed"] = subscribed
|
||||
|
|
|
@ -100,7 +100,7 @@
|
|||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
|
|
@ -72,7 +72,7 @@
|
|||
<div class="video-item {{ view_style }}" id="dl-{{ video.source.youtube_id }}">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video_thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video_thumb">
|
||||
{% if show_ignored_only %}
|
||||
<span>ignored</span>
|
||||
{% else %}
|
||||
|
|
|
@ -12,7 +12,7 @@
|
|||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
@ -98,7 +98,7 @@
|
|||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
|
|
@ -102,7 +102,7 @@
|
|||
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap {{ view_style }}">
|
||||
<div class="video-thumb">
|
||||
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
<img src="{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||
{% if video.source.player.progress %}
|
||||
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||
{% else %}
|
||||
|
|
|
@ -2,8 +2,10 @@
|
|||
{% block content %}
|
||||
{% load static %}
|
||||
{% load humanize %}
|
||||
<div class="video-main">
|
||||
<div id="player" class="player-wrapper">
|
||||
<div class="video-main">
|
||||
<div class="video-modal"><span class="video-modal-text"></span></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="notifications" id="notifications"></div>
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
|
@ -69,15 +71,24 @@
|
|||
<p class="thumb-icon"><img class="dislike" src="{% static 'img/icon-thumb.svg' %}" alt="thumbs-down">: {{ video.stats.dislike_count|intcomma }}</p>
|
||||
{% endif %}
|
||||
{% if video.stats.average_rating %}
|
||||
<p class="rating-stars">Rating:
|
||||
<div class="rating-stars">
|
||||
{% for star in video.stats.average_rating %}
|
||||
<img src="/static/img/icon-star-{{ star }}.svg" alt="{{ star }}">
|
||||
{% endfor %}
|
||||
</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if video.tags %}
|
||||
<div class="description-box">
|
||||
<div class="video-tag-box">
|
||||
{% for tag in video.tags %}
|
||||
<span class="video-tag">{{ tag }}</span>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if video.description %}
|
||||
<div class="description-box">
|
||||
<p id="text-expand" class="description-text">
|
||||
|
@ -123,6 +134,12 @@
|
|||
</div>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<div class="description-box">
|
||||
<h3>Similar Videos</h3>
|
||||
<div class="video-list grid grid-3" id="similar-videos">
|
||||
</div>
|
||||
<script>getSimilarVideos('{{ video.youtube_id }}')</script>
|
||||
</div>
|
||||
{% if video.comment_count %}
|
||||
<div class="comments-section">
|
||||
<h3>Comments: {{video.comment_count}}</h3>
|
||||
|
|
|
@ -783,6 +783,18 @@ video:-webkit-full-screen {
|
|||
display: flex;
|
||||
}
|
||||
|
||||
.video-tag-box {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.video-tag {
|
||||
padding: 5px 10px;
|
||||
margin: 5px;
|
||||
border: 1px solid var(--accent-font-light);
|
||||
}
|
||||
|
||||
.thumb-icon img,
|
||||
.rating-stars img {
|
||||
width: 20px;
|
||||
|
@ -826,10 +838,13 @@ video:-webkit-full-screen {
|
|||
|
||||
.comment-box {
|
||||
padding-bottom: 1rem;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.comments-replies {
|
||||
padding-left: 3rem;
|
||||
display: none;
|
||||
padding-left: 1rem;
|
||||
border-left: 1px solid var(--accent-font-light);
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
|
|
|
@ -793,7 +793,11 @@ function apiRequest(apiEndpoint, method, data) {
|
|||
xhttp.setRequestHeader('Authorization', 'Token ' + sessionToken);
|
||||
xhttp.setRequestHeader('Content-Type', 'application/json');
|
||||
xhttp.send(JSON.stringify(data));
|
||||
if (xhttp.status === 404) {
|
||||
return false;
|
||||
} else {
|
||||
return JSON.parse(xhttp.responseText);
|
||||
}
|
||||
}
|
||||
|
||||
// Gets origin URL
|
||||
|
@ -951,7 +955,7 @@ function createVideo(video, viewStyle) {
|
|||
// create video item div from template
|
||||
const videoId = video.youtube_id;
|
||||
// const mediaUrl = video.media_url;
|
||||
const thumbUrl = '/cache/' + video.vid_thumb_url;
|
||||
// const thumbUrl = '/cache/' + video.vid_thumb_url;
|
||||
const videoTitle = video.title;
|
||||
const videoPublished = video.published;
|
||||
const videoDuration = video.player.duration_str;
|
||||
|
@ -968,7 +972,7 @@ function createVideo(video, viewStyle) {
|
|||
<a href="#player" data-id="${videoId}" onclick="createPlayer(this)">
|
||||
<div class="video-thumb-wrap ${viewStyle}">
|
||||
<div class="video-thumb">
|
||||
<img src="${thumbUrl}" alt="video-thumb">
|
||||
<img src="${video.vid_thumb_url}" alt="video-thumb">
|
||||
</div>
|
||||
<div class="video-play">
|
||||
<img src="/static/img/icon-play.svg" alt="play-icon">
|
||||
|
@ -1123,12 +1127,18 @@ function writeComments(allComments) {
|
|||
if (rootComment.comment_replies) {
|
||||
let commentReplyBox = document.createElement('div');
|
||||
commentReplyBox.setAttribute('class', 'comments-replies');
|
||||
for (let j = 0; j < rootComment.comment_replies.length; j++) {
|
||||
commentReplyBox.setAttribute('id', rootComment.comment_id + '-replies');
|
||||
let totalReplies = rootComment.comment_replies.length;
|
||||
if (totalReplies > 0) {
|
||||
let replyButton = createReplyButton(rootComment.comment_id + '-replies', totalReplies);
|
||||
commentBox.appendChild(replyButton);
|
||||
}
|
||||
for (let j = 0; j < totalReplies; j++) {
|
||||
const commentReply = rootComment.comment_replies[j];
|
||||
let commentReplyDiv = createCommentBox(commentReply, false);
|
||||
commentReplyBox.appendChild(commentReplyDiv);
|
||||
}
|
||||
if (rootComment.comment_replies.length > 0) {
|
||||
if (totalReplies > 0) {
|
||||
commentBox.appendChild(commentReplyBox);
|
||||
}
|
||||
}
|
||||
|
@ -1136,6 +1146,27 @@ function writeComments(allComments) {
|
|||
}
|
||||
}
|
||||
|
||||
function createReplyButton(replyId, totalReplies) {
|
||||
let replyButton = document.createElement('button');
|
||||
replyButton.innerHTML = `<span id="toggle-icon">+</span> ${totalReplies} replies`;
|
||||
replyButton.setAttribute('data-id', replyId);
|
||||
replyButton.setAttribute('onclick', 'toggleCommentReplies(this)');
|
||||
return replyButton;
|
||||
}
|
||||
|
||||
function toggleCommentReplies(button) {
|
||||
let commentReplyId = button.getAttribute('data-id');
|
||||
let state = document.getElementById(commentReplyId).style.display;
|
||||
|
||||
if (state === 'none' || state === '') {
|
||||
document.getElementById(commentReplyId).style.display = 'block';
|
||||
button.querySelector('#toggle-icon').innerHTML = '-';
|
||||
} else {
|
||||
document.getElementById(commentReplyId).style.display = 'none';
|
||||
button.querySelector('#toggle-icon').innerHTML = '+';
|
||||
}
|
||||
}
|
||||
|
||||
function createCommentBox(comment, isRoot) {
|
||||
let commentBox = document.createElement('div');
|
||||
commentBox.setAttribute('class', 'comment-box');
|
||||
|
@ -1167,7 +1198,7 @@ function createCommentBox(comment, isRoot) {
|
|||
commentMeta.innerHTML = `<span>${comment.comment_time_text}</span>`;
|
||||
|
||||
if (comment.comment_likecount > 0) {
|
||||
let numberFormatted = formatNumbers(comment.comment_likecount)
|
||||
let numberFormatted = formatNumbers(comment.comment_likecount);
|
||||
commentMeta.innerHTML += `${spacer}<span class="thumb-icon"><img src="/static/img/icon-thumb.svg"> ${numberFormatted}</span>`;
|
||||
}
|
||||
|
||||
|
@ -1180,6 +1211,35 @@ function createCommentBox(comment, isRoot) {
|
|||
return commentBox;
|
||||
}
|
||||
|
||||
function getSimilarVideos(videoId) {
|
||||
let apiEndpoint = '/api/video/' + videoId + '/similar/';
|
||||
let response = apiRequest(apiEndpoint, 'GET');
|
||||
if (!response) {
|
||||
populateEmpty();
|
||||
return;
|
||||
}
|
||||
let allSimilar = response.data;
|
||||
if (allSimilar.length > 0) {
|
||||
populateSimilar(allSimilar);
|
||||
}
|
||||
}
|
||||
|
||||
function populateSimilar(allSimilar) {
|
||||
let similarBox = document.getElementById('similar-videos');
|
||||
for (let i = 0; i < allSimilar.length; i++) {
|
||||
const similarRaw = allSimilar[i];
|
||||
let similarDiv = createVideo(similarRaw, 'grid');
|
||||
similarBox.appendChild(similarDiv);
|
||||
}
|
||||
}
|
||||
|
||||
function populateEmpty() {
|
||||
let similarBox = document.getElementById('similar-videos');
|
||||
let emptyMessage = document.createElement('p');
|
||||
emptyMessage.innerText = 'No similar videos found.';
|
||||
similarBox.appendChild(emptyMessage);
|
||||
}
|
||||
|
||||
// generic
|
||||
|
||||
function sendPost(payload) {
|
||||
|
|
Loading…
Reference in New Issue