improved sponsorblock indexing, #build

Changed:
- improved more flexible sb timestamps indexing
- fixed per channel sb activate/deactivate
- improved Dockerfile with multi stage build
- added playlist API list view
This commit is contained in:
simon 2022-04-13 15:56:05 +07:00
commit c59c1fc007
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
12 changed files with 157 additions and 128 deletions

View File

@ -1,17 +1,13 @@
# build the tube archivist image from default python slim image # multi stage to build tube archivist
# first stage to build python wheel, copy into final image
FROM python:3.10.4-slim-bullseye
# First stage to build python wheel
FROM python:3.10.4-slim-bullseye AS builder
ARG TARGETPLATFORM ARG TARGETPLATFORM
ARG INSTALL_DEBUG
ENV PYTHONUNBUFFERED 1 RUN apt-get update
RUN apt-get install -y --no-install-recommends build-essential gcc curl
# install distro packages needed
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
build-essential \
nginx \
atomicparsley \
curl && rm -rf /var/lib/apt/lists/*
# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64 # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
@ -27,11 +23,35 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \ apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
; fi ; fi
# install requirements
COPY ./tubearchivist/requirements.txt /requirements.txt
RUN pip install --user -r requirements.txt
# build final image
FROM python:3.10.4-slim-bullseye as tubearchivist
ARG TARGETPLATFORM
ARG INSTALL_DEBUG
ENV PYTHONUNBUFFERED 1
# copy build requirements
COPY --from=builder /root/.local /root/.local
COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg
COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe
ENV PATH=/root/.local/bin:$PATH
# install distro packages needed
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
nginx \
atomicparsley \
curl && rm -rf /var/lib/apt/lists/*
# install debug tools for testing environment # install debug tools for testing environment
RUN if [ "$INSTALL_DEBUG" ] ; then \ RUN if [ "$INSTALL_DEBUG" ] ; then \
apt-get -y update && apt-get -y install --no-install-recommends \ apt-get -y update && apt-get -y install --no-install-recommends \
vim htop bmon net-tools iputils-ping procps \ vim htop bmon net-tools iputils-ping procps \
&& pip install --no-cache-dir ipython --src /usr/local/src \ && pip install --user ipython \
; fi ; fi
# make folders # make folders
@ -39,12 +59,6 @@ RUN mkdir /cache
RUN mkdir /youtube RUN mkdir /youtube
RUN mkdir /app RUN mkdir /app
# install python dependencies
COPY ./tubearchivist/requirements.txt /requirements.txt
RUN pip install --upgrade pip && \
pip install --upgrade setuptools && \
pip install --no-cache-dir -r requirements.txt --src /usr/local/src
# copy config files # copy config files
COPY docker_assets/nginx.conf /etc/nginx/sites-available/default COPY docker_assets/nginx.conf /etc/nginx/sites-available/default

View File

@ -123,6 +123,9 @@ POST /api/channel/
## Channel Item View ## Channel Item View
/api/channel/\<channel_id>/ /api/channel/\<channel_id>/
## Playlist List View
/api/playlist/
## Playlists Item View ## Playlists Item View
/api/playlist/\<playlist_id>/ /api/playlist/\<playlist_id>/

View File

@ -40,6 +40,8 @@ class SearchProcess:
processed = self._process_video(result["_source"]) processed = self._process_video(result["_source"])
if index == "ta_channel": if index == "ta_channel":
processed = self._process_channel(result["_source"]) processed = self._process_channel(result["_source"])
if index == "ta_playlist":
processed = self._process_playlist(result["_source"])
return processed return processed
@ -80,3 +82,19 @@ class SearchProcess:
) )
return dict(sorted(video_dict.items())) return dict(sorted(video_dict.items()))
@staticmethod
def _process_playlist(playlist_dict):
"""run on single playlist dict"""
playlist_id = playlist_dict["playlist_id"]
playlist_last_refresh = date_praser(
playlist_dict["playlist_last_refresh"]
)
playlist_dict.update(
{
"playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
"playlist_last_refresh": playlist_last_refresh,
}
)
return dict(sorted(playlist_dict.items()))

View File

@ -7,6 +7,7 @@ from api.views import (
DownloadApiView, DownloadApiView,
LoginApiView, LoginApiView,
PingView, PingView,
PlaylistApiListView,
PlaylistApiView, PlaylistApiView,
VideoApiListView, VideoApiListView,
VideoApiView, VideoApiView,
@ -53,6 +54,11 @@ urlpatterns = [
PlaylistApiView.as_view(), PlaylistApiView.as_view(),
name="api-playlist", name="api-playlist",
), ),
path(
"playlist/",
PlaylistApiListView.as_view(),
name="api-playlist-list",
),
path( path(
"download/", "download/",
DownloadApiListView.as_view(), DownloadApiListView.as_view(),

View File

@ -258,6 +258,22 @@ class PlaylistApiView(ApiBaseView):
return Response(self.response, status=self.status_code) return Response(self.response, status=self.status_code)
class PlaylistApiListView(ApiBaseView):
"""resolves to /api/playlist/
GET: returns list of indexed playlists
"""
search_base = "ta_playlist/_search/"
def get(self, request):
# pylint: disable=unused-argument
"""handle get request"""
data = {"query": {"match_all": {}}}
self.get_document_list(data)
self.get_paginate()
return Response(self.response)
class DownloadApiView(ApiBaseView): class DownloadApiView(ApiBaseView):
"""resolves to /api/download/<video_id>/ """resolves to /api/download/<video_id>/
GET: returns metadata dict of an item in the download queue GET: returns metadata dict of an item in the download queue

View File

@ -218,6 +218,19 @@
"index": false "index": false
} }
} }
},
"sponsorblock": {
"properties": {
"last_refresh": {
"type": "date"
},
"has_unlocked": {
"type": "boolean"
},
"is_enabled": {
"type": "boolean"
}
}
} }
}, },
"expected_set": { "expected_set": {

View File

@ -200,7 +200,7 @@ class ChannelOverwriteForm(forms.Form):
SP_CHOICES = [ SP_CHOICES = [
("", "-- change sponsorblock integrations"), ("", "-- change sponsorblock integrations"),
("0", "disable sponsorblock integration"), ("disable", "disable sponsorblock integration"),
("1", "enable sponsorblock integration"), ("1", "enable sponsorblock integration"),
] ]

View File

@ -351,6 +351,9 @@ class YoutubeChannel(YouTubeItem):
for key, value in overwrites.items(): for key, value in overwrites.items():
if key not in valid_keys: if key not in valid_keys:
raise ValueError(f"invalid overwrite key: {key}") raise ValueError(f"invalid overwrite key: {key}")
if value == "disable":
to_write[key] = False
continue
if value in [0, "0"]: if value in [0, "0"]:
del to_write[key] del to_write[key]
continue continue

View File

@ -292,6 +292,7 @@ class SponsorBlock:
def __init__(self, user_id=False): def __init__(self, user_id=False):
self.user_id = user_id self.user_id = user_id
self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
self.last_refresh = int(datetime.now().strftime("%s"))
def get_sb_id(self): def get_sb_id(self):
"""get sponsorblock userid or generate if needed""" """get sponsorblock userid or generate if needed"""
@ -315,9 +316,35 @@ class SponsorBlock:
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
if not response.ok: if not response.ok:
print(f"{youtube_id}: sponsorblock failed: {response.text}") print(f"{youtube_id}: sponsorblock failed: {response.text}")
return False sponsor_dict = {
"last_refresh": self.last_refresh,
"is_enabled": True,
"segments": [],
}
else:
all_segments = response.json()
sponsor_dict = self._get_sponsor_dict(all_segments)
return response.json() return sponsor_dict
def _get_sponsor_dict(self, all_segments):
"""format and process response"""
has_unlocked = False
cleaned_segments = []
for segment in all_segments:
if not segment["locked"]:
has_unlocked = True
del segment["userID"]
del segment["description"]
cleaned_segments.append(segment)
sponsor_dict = {
"last_refresh": self.last_refresh,
"has_unlocked": has_unlocked,
"is_enabled": True,
"segments": cleaned_segments,
}
return sponsor_dict
def post_timestamps(self, youtube_id, start_time, end_time): def post_timestamps(self, youtube_id, start_time, end_time):
"""post timestamps to api""" """post timestamps to api"""
@ -385,16 +412,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
def _check_get_sb(self): def _check_get_sb(self):
"""check if need to run sponsor block""" """check if need to run sponsor block"""
integrate = False
if self.config["downloads"]["integrate_sponsorblock"]: if self.config["downloads"]["integrate_sponsorblock"]:
return True integrate = True
try:
single_overwrite = self.video_overwrites[self.youtube_id]
_ = single_overwrite["integrate_sponsorblock"]
return True
except KeyError:
return False
return False if self.video_overwrites:
single_overwrite = self.video_overwrites.get(self.youtube_id)
if not single_overwrite:
return integrate
integrate = single_overwrite.get("integrate_sponsorblock", False)
return integrate
def _process_youtube_meta(self): def _process_youtube_meta(self):
"""extract relevant fields from youtube""" """extract relevant fields from youtube"""

View File

@ -5,31 +5,11 @@
<div class="video-main"></div> <div class="video-main"></div>
<div class="notifications" id="notifications"></div> <div class="notifications" id="notifications"></div>
<div class="sponsorblock" id="sponsorblock"> <div class="sponsorblock" id="sponsorblock">
{% if video.channel.channel_overwrites.integrate_sponsorblock %} {% if video.sponsorblock.is_enabled %}
{% if video.channel.channel_overwrites.integrate_sponsorblock == True %} {% if video.sponsorblock.segments|length == 0 %}
{% if not video.sponsorblock %}
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4> <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
{% endif %} {% elif video.sponsorblock.has_unlocked %}
{% if video.sponsorblock %}
{% for segment in video.sponsorblock %}
{% if segment.locked != 1 %}
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4> <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
{{ break }}
{% endif %}
{% endfor %}
{% endif %}
{% endif %}
{% elif config.downloads.integrate_sponsorblock %}
{% if not video.sponsorblock %}
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
{% endif %}
{% if video.sponsorblock %}
{% for segment in video.sponsorblock %}
{% if segment.locked != 1 %}
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
{{ break }}
{% endif %}
{% endfor %}
{% endif %} {% endif %}
{% endif %} {% endif %}
</div> </div>

View File

@ -1,6 +1,6 @@
beautifulsoup4==4.11.1 beautifulsoup4==4.11.1
celery==5.2.6 celery==5.2.6
Django==4.0.3 Django==4.0.4
django-cors-headers==3.11.0 django-cors-headers==3.11.0
djangorestframework==3.13.1 djangorestframework==3.13.1
Pillow==9.1.0 Pillow==9.1.0

View File

@ -333,24 +333,21 @@ function createPlayer(button) {
var videoData = getVideoData(videoId); var videoData = getVideoData(videoId);
var sponsorBlockElements = ''; var sponsorBlockElements = '';
if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) { if (videoData.data.sponsorblock.is_enabled) {
sponsorBlock = videoData.data.sponsorblock; sponsorBlock = videoData.data.sponsorblock;
if (!sponsorBlock) { if (sponsorBlock.segments.length == 0) {
sponsorBlockElements = ` sponsorBlockElements = `
<div class="sponsorblock" id="sponsorblock"> <div class="sponsorblock" id="sponsorblock">
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on Youtube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4> <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on Youtube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
</div> </div>
`; `;
} else { } else {
for(let i in sponsorBlock) { if (sponsorBlock.has_unlocked) {
if(sponsorBlock[i].locked != 1) {
sponsorBlockElements = ` sponsorBlockElements = `
<div class="sponsorblock" id="sponsorblock"> <div class="sponsorblock" id="sponsorblock">
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4> <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
</div> </div>
`; `;
break;
}
} }
} }
} }
@ -425,53 +422,6 @@ function createPlayer(button) {
divPlayer.innerHTML = markup; divPlayer.innerHTML = markup;
} }
// function sendSponsorBlockVote(uuid, vote) {
// var videoId = getVideoPlayerVideoId();
// postSponsorSegmentVote(videoId, uuid, vote);
// }
// var sponsorBlockTimestamps = [];
// function sendSponsorBlockSegment() {
// var videoId = getVideoPlayerVideoId();
// var currentTime = getVideoPlayerCurrentTime();
// var sponsorBlockElement = document.getElementById("sponsorblock");
// if (sponsorBlockTimestamps[1]) {
// if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) {
// postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]);
// sponsorBlockElement.innerHTML = `
// <p>Timestamps sent! (Not really)</p>
// `;
// setTimeout(function(){
// sponsorBlockElement.innerHTML = `
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
// `;
// }, 3000);
// } else {
// sponsorBlockElement.innerHTML = `
// <span class="danger-zone">Invalid Timestamps!</span>
// `;
// setTimeout(function(){
// sponsorBlockElement.innerHTML = `
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
// `;
// }, 3000);
// }
// sponsorBlockTimestamps = [];
// } else if (sponsorBlockTimestamps[0]) {
// sponsorBlockTimestamps.push(currentTime);
// sponsorBlockElement.innerHTML = `
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[0]}'"><p>${sponsorBlockTimestamps[0].toFixed(1)} s | </p>
// <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[1]}'"><p>${sponsorBlockTimestamps[1].toFixed(1)} s | </p>
// <img src="/static/img/PlayerUploadIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Confirm</button>
// `;
// } else {
// sponsorBlockTimestamps.push(currentTime);
// sponsorBlockElement.innerHTML = `
// <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">End</button>
// `;
// }
// }
// Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)` // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)`
function insertVideoTag(videoData, videoProgress) { function insertVideoTag(videoData, videoProgress) {
var videoTag = createVideoTag(videoData, videoProgress); var videoTag = createVideoTag(videoData, videoProgress);
@ -563,23 +513,14 @@ function onVideoProgress() {
var videoElement = getVideoPlayer(); var videoElement = getVideoPlayer();
// var sponsorBlockElement = document.getElementById("sponsorblock"); // var sponsorBlockElement = document.getElementById("sponsorblock");
var notificationsElement = document.getElementById("notifications"); var notificationsElement = document.getElementById("notifications");
if (sponsorBlock) { if (sponsorBlock.segments.length > 0) {
for(let i in sponsorBlock) { for(let i in sponsorBlock.segments) {
if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) { if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) {
videoElement.currentTime = sponsorBlock[i].segment[1]; videoElement.currentTime = sponsorBlock.segments[i].segment[1];
notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.</h3>`; notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock.segments[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.</h3>`;
} }
// if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) { if(currentTime > sponsorBlock.segments[i].segment[1] + 10) {
// if(sponsorBlock[i].locked != 1) { var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
// sponsorBlockElement.innerHTML += `
// <div id="${sponsorBlock[i].UUID}">
// <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', 1)">Up Vote</button>
// <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', -1)">Down Vote</button>
// </div>`;
// }
// }
if(currentTime > sponsorBlock[i].segment[1] + 10) {
var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID);
if(notificationsElementUUID) { if(notificationsElementUUID) {
notificationsElementUUID.outerHTML = ''; notificationsElementUUID.outerHTML = '';
} }
@ -602,6 +543,12 @@ function onVideoEnded() {
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
updateVideoWatchStatus(videoId, "unwatched"); updateVideoWatchStatus(videoId, "unwatched");
} }
for(let i in sponsorBlock.segments) {
var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
if(notificationsElementUUID) {
notificationsElementUUID.outerHTML = '';
}
}
} }
function watchedThreshold(currentTime, duration) { function watchedThreshold(currentTime, duration) {