mirror of
https://github.com/tubearchivist/tubearchivist.git
synced 2024-10-14 01:37:15 +00:00
improved sponsorblock indexing, #build
Changed: - improved more flexible sb timestamps indexing - fixed per channel sb activate/deactivate - improved Dockerfile with multi stage build - added playlist API list view
This commit is contained in:
commit
c59c1fc007
50
Dockerfile
50
Dockerfile
@ -1,17 +1,13 @@
|
||||
# build the tube archivist image from default python slim image
|
||||
# multi stage to build tube archivist
|
||||
# first stage to build python wheel, copy into final image
|
||||
|
||||
FROM python:3.10.4-slim-bullseye
|
||||
|
||||
# First stage to build python wheel
|
||||
FROM python:3.10.4-slim-bullseye AS builder
|
||||
ARG TARGETPLATFORM
|
||||
ARG INSTALL_DEBUG
|
||||
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# install distro packages needed
|
||||
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
|
||||
build-essential \
|
||||
nginx \
|
||||
atomicparsley \
|
||||
curl && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y --no-install-recommends build-essential gcc curl
|
||||
|
||||
# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
|
||||
@ -27,11 +23,35 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
|
||||
apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
|
||||
; fi
|
||||
|
||||
# install requirements
|
||||
COPY ./tubearchivist/requirements.txt /requirements.txt
|
||||
RUN pip install --user -r requirements.txt
|
||||
|
||||
# build final image
|
||||
FROM python:3.10.4-slim-bullseye as tubearchivist
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG INSTALL_DEBUG
|
||||
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# copy build requirements
|
||||
COPY --from=builder /root/.local /root/.local
|
||||
COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg
|
||||
COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe
|
||||
ENV PATH=/root/.local/bin:$PATH
|
||||
|
||||
# install distro packages needed
|
||||
RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
|
||||
nginx \
|
||||
atomicparsley \
|
||||
curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# install debug tools for testing environment
|
||||
RUN if [ "$INSTALL_DEBUG" ] ; then \
|
||||
apt-get -y update && apt-get -y install --no-install-recommends \
|
||||
vim htop bmon net-tools iputils-ping procps \
|
||||
&& pip install --no-cache-dir ipython --src /usr/local/src \
|
||||
&& pip install --user ipython \
|
||||
; fi
|
||||
|
||||
# make folders
|
||||
@ -39,12 +59,6 @@ RUN mkdir /cache
|
||||
RUN mkdir /youtube
|
||||
RUN mkdir /app
|
||||
|
||||
# install python dependencies
|
||||
COPY ./tubearchivist/requirements.txt /requirements.txt
|
||||
RUN pip install --upgrade pip && \
|
||||
pip install --upgrade setuptools && \
|
||||
pip install --no-cache-dir -r requirements.txt --src /usr/local/src
|
||||
|
||||
# copy config files
|
||||
COPY docker_assets/nginx.conf /etc/nginx/sites-available/default
|
||||
|
||||
|
@ -123,6 +123,9 @@ POST /api/channel/
|
||||
## Channel Item View
|
||||
/api/channel/\<channel_id>/
|
||||
|
||||
## Playlist List View
|
||||
/api/playlist/
|
||||
|
||||
## Playlists Item View
|
||||
/api/playlist/\<playlist_id>/
|
||||
|
||||
|
@ -40,6 +40,8 @@ class SearchProcess:
|
||||
processed = self._process_video(result["_source"])
|
||||
if index == "ta_channel":
|
||||
processed = self._process_channel(result["_source"])
|
||||
if index == "ta_playlist":
|
||||
processed = self._process_playlist(result["_source"])
|
||||
|
||||
return processed
|
||||
|
||||
@ -80,3 +82,19 @@ class SearchProcess:
|
||||
)
|
||||
|
||||
return dict(sorted(video_dict.items()))
|
||||
|
||||
@staticmethod
|
||||
def _process_playlist(playlist_dict):
|
||||
"""run on single playlist dict"""
|
||||
playlist_id = playlist_dict["playlist_id"]
|
||||
playlist_last_refresh = date_praser(
|
||||
playlist_dict["playlist_last_refresh"]
|
||||
)
|
||||
playlist_dict.update(
|
||||
{
|
||||
"playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
|
||||
"playlist_last_refresh": playlist_last_refresh,
|
||||
}
|
||||
)
|
||||
|
||||
return dict(sorted(playlist_dict.items()))
|
||||
|
@ -7,6 +7,7 @@ from api.views import (
|
||||
DownloadApiView,
|
||||
LoginApiView,
|
||||
PingView,
|
||||
PlaylistApiListView,
|
||||
PlaylistApiView,
|
||||
VideoApiListView,
|
||||
VideoApiView,
|
||||
@ -53,6 +54,11 @@ urlpatterns = [
|
||||
PlaylistApiView.as_view(),
|
||||
name="api-playlist",
|
||||
),
|
||||
path(
|
||||
"playlist/",
|
||||
PlaylistApiListView.as_view(),
|
||||
name="api-playlist-list",
|
||||
),
|
||||
path(
|
||||
"download/",
|
||||
DownloadApiListView.as_view(),
|
||||
|
@ -258,6 +258,22 @@ class PlaylistApiView(ApiBaseView):
|
||||
return Response(self.response, status=self.status_code)
|
||||
|
||||
|
||||
class PlaylistApiListView(ApiBaseView):
|
||||
"""resolves to /api/playlist/
|
||||
GET: returns list of indexed playlists
|
||||
"""
|
||||
|
||||
search_base = "ta_playlist/_search/"
|
||||
|
||||
def get(self, request):
|
||||
# pylint: disable=unused-argument
|
||||
"""handle get request"""
|
||||
data = {"query": {"match_all": {}}}
|
||||
self.get_document_list(data)
|
||||
self.get_paginate()
|
||||
return Response(self.response)
|
||||
|
||||
|
||||
class DownloadApiView(ApiBaseView):
|
||||
"""resolves to /api/download/<video_id>/
|
||||
GET: returns metadata dict of an item in the download queue
|
||||
|
@ -218,6 +218,19 @@
|
||||
"index": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"sponsorblock": {
|
||||
"properties": {
|
||||
"last_refresh": {
|
||||
"type": "date"
|
||||
},
|
||||
"has_unlocked": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"is_enabled": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected_set": {
|
||||
|
@ -200,7 +200,7 @@ class ChannelOverwriteForm(forms.Form):
|
||||
|
||||
SP_CHOICES = [
|
||||
("", "-- change sponsorblock integrations"),
|
||||
("0", "disable sponsorblock integration"),
|
||||
("disable", "disable sponsorblock integration"),
|
||||
("1", "enable sponsorblock integration"),
|
||||
]
|
||||
|
||||
|
@ -351,6 +351,9 @@ class YoutubeChannel(YouTubeItem):
|
||||
for key, value in overwrites.items():
|
||||
if key not in valid_keys:
|
||||
raise ValueError(f"invalid overwrite key: {key}")
|
||||
if value == "disable":
|
||||
to_write[key] = False
|
||||
continue
|
||||
if value in [0, "0"]:
|
||||
del to_write[key]
|
||||
continue
|
||||
|
@ -292,6 +292,7 @@ class SponsorBlock:
|
||||
def __init__(self, user_id=False):
|
||||
self.user_id = user_id
|
||||
self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
|
||||
self.last_refresh = int(datetime.now().strftime("%s"))
|
||||
|
||||
def get_sb_id(self):
|
||||
"""get sponsorblock userid or generate if needed"""
|
||||
@ -315,9 +316,35 @@ class SponsorBlock:
|
||||
response = requests.get(url, headers=headers)
|
||||
if not response.ok:
|
||||
print(f"{youtube_id}: sponsorblock failed: {response.text}")
|
||||
return False
|
||||
sponsor_dict = {
|
||||
"last_refresh": self.last_refresh,
|
||||
"is_enabled": True,
|
||||
"segments": [],
|
||||
}
|
||||
else:
|
||||
all_segments = response.json()
|
||||
sponsor_dict = self._get_sponsor_dict(all_segments)
|
||||
|
||||
return response.json()
|
||||
return sponsor_dict
|
||||
|
||||
def _get_sponsor_dict(self, all_segments):
|
||||
"""format and process response"""
|
||||
has_unlocked = False
|
||||
cleaned_segments = []
|
||||
for segment in all_segments:
|
||||
if not segment["locked"]:
|
||||
has_unlocked = True
|
||||
del segment["userID"]
|
||||
del segment["description"]
|
||||
cleaned_segments.append(segment)
|
||||
|
||||
sponsor_dict = {
|
||||
"last_refresh": self.last_refresh,
|
||||
"has_unlocked": has_unlocked,
|
||||
"is_enabled": True,
|
||||
"segments": cleaned_segments,
|
||||
}
|
||||
return sponsor_dict
|
||||
|
||||
def post_timestamps(self, youtube_id, start_time, end_time):
|
||||
"""post timestamps to api"""
|
||||
@ -385,16 +412,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||
|
||||
def _check_get_sb(self):
|
||||
"""check if need to run sponsor block"""
|
||||
integrate = False
|
||||
if self.config["downloads"]["integrate_sponsorblock"]:
|
||||
return True
|
||||
try:
|
||||
single_overwrite = self.video_overwrites[self.youtube_id]
|
||||
_ = single_overwrite["integrate_sponsorblock"]
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
integrate = True
|
||||
|
||||
return False
|
||||
if self.video_overwrites:
|
||||
single_overwrite = self.video_overwrites.get(self.youtube_id)
|
||||
if not single_overwrite:
|
||||
return integrate
|
||||
|
||||
integrate = single_overwrite.get("integrate_sponsorblock", False)
|
||||
|
||||
return integrate
|
||||
|
||||
def _process_youtube_meta(self):
|
||||
"""extract relevant fields from youtube"""
|
||||
|
@ -5,31 +5,11 @@
|
||||
<div class="video-main"></div>
|
||||
<div class="notifications" id="notifications"></div>
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
{% if video.channel.channel_overwrites.integrate_sponsorblock %}
|
||||
{% if video.channel.channel_overwrites.integrate_sponsorblock == True %}
|
||||
{% if not video.sponsorblock %}
|
||||
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
{% endif %}
|
||||
{% if video.sponsorblock %}
|
||||
{% for segment in video.sponsorblock %}
|
||||
{% if segment.locked != 1 %}
|
||||
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
{{ break }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% elif config.downloads.integrate_sponsorblock %}
|
||||
{% if not video.sponsorblock %}
|
||||
{% if video.sponsorblock.is_enabled %}
|
||||
{% if video.sponsorblock.segments|length == 0 %}
|
||||
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
{% endif %}
|
||||
{% if video.sponsorblock %}
|
||||
{% for segment in video.sponsorblock %}
|
||||
{% if segment.locked != 1 %}
|
||||
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
{{ break }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% elif video.sponsorblock.has_unlocked %}
|
||||
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
@ -1,6 +1,6 @@
|
||||
beautifulsoup4==4.11.1
|
||||
celery==5.2.6
|
||||
Django==4.0.3
|
||||
Django==4.0.4
|
||||
django-cors-headers==3.11.0
|
||||
djangorestframework==3.13.1
|
||||
Pillow==9.1.0
|
||||
|
@ -333,24 +333,21 @@ function createPlayer(button) {
|
||||
var videoData = getVideoData(videoId);
|
||||
|
||||
var sponsorBlockElements = '';
|
||||
if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) {
|
||||
if (videoData.data.sponsorblock.is_enabled) {
|
||||
sponsorBlock = videoData.data.sponsorblock;
|
||||
if (!sponsorBlock) {
|
||||
if (sponsorBlock.segments.length == 0) {
|
||||
sponsorBlockElements = `
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
<h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on Youtube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
</div>
|
||||
`;
|
||||
} else {
|
||||
for(let i in sponsorBlock) {
|
||||
if(sponsorBlock[i].locked != 1) {
|
||||
sponsorBlockElements = `
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
</div>
|
||||
`;
|
||||
break;
|
||||
}
|
||||
if (sponsorBlock.has_unlocked) {
|
||||
sponsorBlockElements = `
|
||||
<div class="sponsorblock" id="sponsorblock">
|
||||
<h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -425,53 +422,6 @@ function createPlayer(button) {
|
||||
divPlayer.innerHTML = markup;
|
||||
}
|
||||
|
||||
// function sendSponsorBlockVote(uuid, vote) {
|
||||
// var videoId = getVideoPlayerVideoId();
|
||||
// postSponsorSegmentVote(videoId, uuid, vote);
|
||||
// }
|
||||
|
||||
// var sponsorBlockTimestamps = [];
|
||||
// function sendSponsorBlockSegment() {
|
||||
// var videoId = getVideoPlayerVideoId();
|
||||
// var currentTime = getVideoPlayerCurrentTime();
|
||||
// var sponsorBlockElement = document.getElementById("sponsorblock");
|
||||
// if (sponsorBlockTimestamps[1]) {
|
||||
// if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) {
|
||||
// postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]);
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <p>Timestamps sent! (Not really)</p>
|
||||
// `;
|
||||
// setTimeout(function(){
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
|
||||
// `;
|
||||
// }, 3000);
|
||||
// } else {
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <span class="danger-zone">Invalid Timestamps!</span>
|
||||
// `;
|
||||
// setTimeout(function(){
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
|
||||
// `;
|
||||
// }, 3000);
|
||||
// }
|
||||
// sponsorBlockTimestamps = [];
|
||||
// } else if (sponsorBlockTimestamps[0]) {
|
||||
// sponsorBlockTimestamps.push(currentTime);
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[0]}'"><p>${sponsorBlockTimestamps[0].toFixed(1)} s | </p>
|
||||
// <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[1]}'"><p>${sponsorBlockTimestamps[1].toFixed(1)} s | </p>
|
||||
// <img src="/static/img/PlayerUploadIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Confirm</button>
|
||||
// `;
|
||||
// } else {
|
||||
// sponsorBlockTimestamps.push(currentTime);
|
||||
// sponsorBlockElement.innerHTML = `
|
||||
// <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">End</button>
|
||||
// `;
|
||||
// }
|
||||
// }
|
||||
|
||||
// Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)`
|
||||
function insertVideoTag(videoData, videoProgress) {
|
||||
var videoTag = createVideoTag(videoData, videoProgress);
|
||||
@ -563,23 +513,14 @@ function onVideoProgress() {
|
||||
var videoElement = getVideoPlayer();
|
||||
// var sponsorBlockElement = document.getElementById("sponsorblock");
|
||||
var notificationsElement = document.getElementById("notifications");
|
||||
if (sponsorBlock) {
|
||||
for(let i in sponsorBlock) {
|
||||
if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) {
|
||||
videoElement.currentTime = sponsorBlock[i].segment[1];
|
||||
notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.</h3>`;
|
||||
if (sponsorBlock.segments.length > 0) {
|
||||
for(let i in sponsorBlock.segments) {
|
||||
if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) {
|
||||
videoElement.currentTime = sponsorBlock.segments[i].segment[1];
|
||||
notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock.segments[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.</h3>`;
|
||||
}
|
||||
// if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) {
|
||||
// if(sponsorBlock[i].locked != 1) {
|
||||
// sponsorBlockElement.innerHTML += `
|
||||
// <div id="${sponsorBlock[i].UUID}">
|
||||
// <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', 1)">Up Vote</button>
|
||||
// <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', -1)">Down Vote</button>
|
||||
// </div>`;
|
||||
// }
|
||||
// }
|
||||
if(currentTime > sponsorBlock[i].segment[1] + 10) {
|
||||
var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID);
|
||||
if(currentTime > sponsorBlock.segments[i].segment[1] + 10) {
|
||||
var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
|
||||
if(notificationsElementUUID) {
|
||||
notificationsElementUUID.outerHTML = '';
|
||||
}
|
||||
@ -602,6 +543,12 @@ function onVideoEnded() {
|
||||
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
|
||||
updateVideoWatchStatus(videoId, "unwatched");
|
||||
}
|
||||
for(let i in sponsorBlock.segments) {
|
||||
var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
|
||||
if(notificationsElementUUID) {
|
||||
notificationsElementUUID.outerHTML = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function watchedThreshold(currentTime, duration) {
|
||||
|
Loading…
Reference in New Issue
Block a user