improved sponsorblock indexing, #build

Changed: - improved more flexible sb timestamps indexing - fixed per channel sb activate/deactivate - improved Dockerfile with multi stage build - added playlist API list view
2022-04-13 15:56:05 +07:00 · 2022-04-13 15:56:05 +07:00 · c59c1fc007
parent a8ee0da2b8 3007e02fe5
commit c59c1fc007
12 changed files with 157 additions and 128 deletions
--- a/50
+++ b/50
@ -1,17 +1,13 @@
-# build the tube archivist image from default python slim image
+# multi stage to build tube archivist
+# first stage to build python wheel, copy into final image

-FROM python:3.10.4-slim-bullseye
+
+# First stage to build python wheel
+FROM python:3.10.4-slim-bullseye AS builder
 ARG TARGETPLATFORM
-ARG INSTALL_DEBUG

-ENV PYTHONUNBUFFERED 1
-
-# install distro packages needed
-RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
-    build-essential \
-    nginx \
-    atomicparsley \
-    curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends build-essential gcc curl

 # get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
@ -27,11 +23,35 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
        apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \
    ; fi

+# install requirements
+COPY ./tubearchivist/requirements.txt /requirements.txt
+RUN pip install --user -r requirements.txt
+
+# build final image
+FROM python:3.10.4-slim-bullseye as tubearchivist
+
+ARG TARGETPLATFORM
+ARG INSTALL_DEBUG
+
+ENV PYTHONUNBUFFERED 1
+
+# copy build requirements
+COPY --from=builder /root/.local /root/.local
+COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg
+COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe
+ENV PATH=/root/.local/bin:$PATH
+
+# install distro packages needed
+RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \
+    nginx \
+    atomicparsley \
+    curl && rm -rf /var/lib/apt/lists/*
+
 # install debug tools for testing environment
 RUN if [ "$INSTALL_DEBUG" ] ; then \
        apt-get -y update && apt-get -y install --no-install-recommends \
        vim htop bmon net-tools iputils-ping procps \
-        && pip install --no-cache-dir ipython --src /usr/local/src \
+        && pip install --user ipython \
    ; fi

 # make folders
@ -39,12 +59,6 @@ RUN mkdir /cache
 RUN mkdir /youtube
 RUN mkdir /app

-# install python dependencies
-COPY ./tubearchivist/requirements.txt /requirements.txt
-RUN pip install --upgrade pip && \
-    pip install --upgrade setuptools && \
-    pip install --no-cache-dir -r requirements.txt --src /usr/local/src
-
 # copy config files
 COPY docker_assets/nginx.conf /etc/nginx/sites-available/default

--- a/tubearchivist/api/README.md
+++ b/tubearchivist/api/README.md
@ -123,6 +123,9 @@ POST /api/channel/
 ## Channel Item View
 /api/channel/\<channel_id>/

+## Playlist List View
+/api/playlist/
+
 ## Playlists Item View
 /api/playlist/\<playlist_id>/

--- a/tubearchivist/api/src/search_processor.py
+++ b/tubearchivist/api/src/search_processor.py
@ -40,6 +40,8 @@ class SearchProcess:
            processed = self._process_video(result["_source"])
        if index == "ta_channel":
            processed = self._process_channel(result["_source"])
+        if index == "ta_playlist":
+            processed = self._process_playlist(result["_source"])

        return processed

@ -80,3 +82,19 @@ class SearchProcess:
        )

        return dict(sorted(video_dict.items()))
+
+    @staticmethod
+    def _process_playlist(playlist_dict):
+        """run on single playlist dict"""
+        playlist_id = playlist_dict["playlist_id"]
+        playlist_last_refresh = date_praser(
+            playlist_dict["playlist_last_refresh"]
+        )
+        playlist_dict.update(
+            {
+                "playlist_thumbnail": f"/cache/playlists/{playlist_id}.jpg",
+                "playlist_last_refresh": playlist_last_refresh,
+            }
+        )
+
+        return dict(sorted(playlist_dict.items()))
--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@ -7,6 +7,7 @@ from api.views import (
    DownloadApiView,
    LoginApiView,
    PingView,
+    PlaylistApiListView,
    PlaylistApiView,
    VideoApiListView,
    VideoApiView,
@ -53,6 +54,11 @@ urlpatterns = [
        PlaylistApiView.as_view(),
        name="api-playlist",
    ),
+    path(
+        "playlist/",
+        PlaylistApiListView.as_view(),
+        name="api-playlist-list",
+    ),
    path(
        "download/",
        DownloadApiListView.as_view(),
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@ -258,6 +258,22 @@ class PlaylistApiView(ApiBaseView):
        return Response(self.response, status=self.status_code)


+class PlaylistApiListView(ApiBaseView):
+    """resolves to /api/playlist/
+    GET: returns list of indexed playlists
+    """
+
+    search_base = "ta_playlist/_search/"
+
+    def get(self, request):
+        # pylint: disable=unused-argument
+        """handle get request"""
+        data = {"query": {"match_all": {}}}
+        self.get_document_list(data)
+        self.get_paginate()
+        return Response(self.response)
+
+
 class DownloadApiView(ApiBaseView):
    """resolves to /api/download/<video_id>/
    GET: returns metadata dict of an item in the download queue
--- a/tubearchivist/home/src/es/index_mapping.json
+++ b/tubearchivist/home/src/es/index_mapping.json
@ -218,6 +218,19 @@
                            "index": false
                        }
                    }
+                },
+                "sponsorblock": {
+                    "properties": {
+                        "last_refresh": {
+                            "type": "date"
+                        },
+                        "has_unlocked": {
+                            "type": "boolean"
+                        },
+                        "is_enabled": {
+                            "type": "boolean"
+                        }
+                    }
                }
            },
            "expected_set": {
--- a/tubearchivist/home/src/frontend/forms.py
+++ b/tubearchivist/home/src/frontend/forms.py
@ -200,7 +200,7 @@ class ChannelOverwriteForm(forms.Form):

    SP_CHOICES = [
        ("", "-- change sponsorblock integrations"),
-        ("0", "disable sponsorblock integration"),
+        ("disable", "disable sponsorblock integration"),
        ("1", "enable sponsorblock integration"),
    ]

--- a/tubearchivist/home/src/index/channel.py
+++ b/tubearchivist/home/src/index/channel.py
@ -351,6 +351,9 @@ class YoutubeChannel(YouTubeItem):
        for key, value in overwrites.items():
            if key not in valid_keys:
                raise ValueError(f"invalid overwrite key: {key}")
+            if value == "disable":
+                to_write[key] = False
+                continue
            if value in [0, "0"]:
                del to_write[key]
                continue
--- a/tubearchivist/home/src/index/video.py
+++ b/tubearchivist/home/src/index/video.py
@ -292,6 +292,7 @@ class SponsorBlock:
    def __init__(self, user_id=False):
        self.user_id = user_id
        self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+        self.last_refresh = int(datetime.now().strftime("%s"))

    def get_sb_id(self):
        """get sponsorblock userid or generate if needed"""
@ -315,9 +316,35 @@ class SponsorBlock:
        response = requests.get(url, headers=headers)
        if not response.ok:
            print(f"{youtube_id}: sponsorblock failed: {response.text}")
-            return False
+            sponsor_dict = {
+                "last_refresh": self.last_refresh,
+                "is_enabled": True,
+                "segments": [],
+            }
+        else:
+            all_segments = response.json()
+            sponsor_dict = self._get_sponsor_dict(all_segments)

-        return response.json()
+        return sponsor_dict
+
+    def _get_sponsor_dict(self, all_segments):
+        """format and process response"""
+        has_unlocked = False
+        cleaned_segments = []
+        for segment in all_segments:
+            if not segment["locked"]:
+                has_unlocked = True
+            del segment["userID"]
+            del segment["description"]
+            cleaned_segments.append(segment)
+
+        sponsor_dict = {
+            "last_refresh": self.last_refresh,
+            "has_unlocked": has_unlocked,
+            "is_enabled": True,
+            "segments": cleaned_segments,
+        }
+        return sponsor_dict

    def post_timestamps(self, youtube_id, start_time, end_time):
        """post timestamps to api"""
@ -385,16 +412,18 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):

    def _check_get_sb(self):
        """check if need to run sponsor block"""
+        integrate = False
        if self.config["downloads"]["integrate_sponsorblock"]:
-            return True
-        try:
-            single_overwrite = self.video_overwrites[self.youtube_id]
-            _ = single_overwrite["integrate_sponsorblock"]
-            return True
-        except KeyError:
-            return False
+            integrate = True

-        return False
+        if self.video_overwrites:
+            single_overwrite = self.video_overwrites.get(self.youtube_id)
+            if not single_overwrite:
+                return integrate
+
+            integrate = single_overwrite.get("integrate_sponsorblock", False)
+
+        return integrate

    def _process_youtube_meta(self):
        """extract relevant fields from youtube"""
--- a/tubearchivist/home/templates/home/video.html
+++ b/tubearchivist/home/templates/home/video.html
@ -5,31 +5,11 @@
 <div class="video-main"></div>
 <div class="notifications" id="notifications"></div>
 <div class="sponsorblock" id="sponsorblock">
-    {% if video.channel.channel_overwrites.integrate_sponsorblock %}
-        {% if video.channel.channel_overwrites.integrate_sponsorblock == True %}
-            {% if not video.sponsorblock %}
-                <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-            {% endif %}
-            {% if video.sponsorblock %}
-                {% for segment in video.sponsorblock %}
-                    {% if segment.locked != 1 %}
-                        <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                        {{ break }}
-                    {% endif %}
-                {% endfor %}
-            {% endif %}
-        {% endif %}
-    {% elif config.downloads.integrate_sponsorblock %}
-        {% if not video.sponsorblock %}
+    {% if video.sponsorblock.is_enabled %}
+        {% if video.sponsorblock.segments|length == 0 %}
            <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-        {% endif %}
-        {% if video.sponsorblock %}
-            {% for segment in video.sponsorblock %}
-                {% if segment.locked != 1 %}
-                    <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                    {{ break }}
-                {% endif %}
-            {% endfor %}
+        {% elif video.sponsorblock.has_unlocked %}
+            <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v={{ video.youtube_id }}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
        {% endif %}
    {% endif %}
 </div>
--- a/tubearchivist/requirements.txt
+++ b/tubearchivist/requirements.txt
@ -1,6 +1,6 @@
 beautifulsoup4==4.11.1
 celery==5.2.6
-Django==4.0.3
+Django==4.0.4
 django-cors-headers==3.11.0
 djangorestframework==3.13.1
 Pillow==9.1.0
--- a/tubearchivist/static/script.js
+++ b/tubearchivist/static/script.js
@ -333,24 +333,21 @@ function createPlayer(button) {
    var videoData = getVideoData(videoId);

    var sponsorBlockElements = '';
-    if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) {
+    if (videoData.data.sponsorblock.is_enabled) {
        sponsorBlock = videoData.data.sponsorblock;
-        if (!sponsorBlock) {
+        if (sponsorBlock.segments.length == 0) {
            sponsorBlockElements = `
            <div class="sponsorblock" id="sponsorblock">
                <h4>This video doesn't have any sponsor segments added. To add a segment go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on Youtube</a></u> and add a segment using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
            </div>
            `;
        } else {
-            for(let i in sponsorBlock) {
-                if(sponsorBlock[i].locked != 1) {
-                    sponsorBlockElements = `
-                    <div class="sponsorblock" id="sponsorblock">
-                        <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
-                    </div>
-                    `;
-                    break;
-                }
+            if (sponsorBlock.has_unlocked) {
+                sponsorBlockElements = `
+                <div class="sponsorblock" id="sponsorblock">
+                    <h4>This video has unlocked sponsor segments. Go to <u><a href="https://www.youtube.com/watch?v=${videoId}">this video on YouTube</a></u> and vote on the segments using the <u><a href="https://sponsor.ajay.app/">SponsorBlock</a></u> extension.</h4>
+                </div>
+                `;
            }
        }
    }
@ -425,53 +422,6 @@ function createPlayer(button) {
    divPlayer.innerHTML = markup;
 }

-// function sendSponsorBlockVote(uuid, vote) {
-//     var videoId = getVideoPlayerVideoId();
-//     postSponsorSegmentVote(videoId, uuid, vote);
-// }
-
-// var sponsorBlockTimestamps = [];
-// function sendSponsorBlockSegment() {
-//     var videoId = getVideoPlayerVideoId();
-//     var currentTime = getVideoPlayerCurrentTime();
-//     var sponsorBlockElement = document.getElementById("sponsorblock");
-//     if (sponsorBlockTimestamps[1]) {
-//         if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) {
-//             postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]);
-//             sponsorBlockElement.innerHTML = `
-//             <p>Timestamps sent! (Not really)</p>
-//             `;
-//             setTimeout(function(){
-//                 sponsorBlockElement.innerHTML = `
-//                 <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
-//                 `;
-//             }, 3000);
-//         } else {
-//             sponsorBlockElement.innerHTML = `
-//             <span class="danger-zone">Invalid Timestamps!</span>
-//             `;
-//             setTimeout(function(){
-//                 sponsorBlockElement.innerHTML = `
-//                 <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Start</button>
-//                 `;
-//             }, 3000);
-//         }
-//         sponsorBlockTimestamps = [];
-//     } else if (sponsorBlockTimestamps[0]) {
-//         sponsorBlockTimestamps.push(currentTime);
-//         sponsorBlockElement.innerHTML = `
-//         <img src="/static/img/PlayerStartIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[0]}'"><p>${sponsorBlockTimestamps[0].toFixed(1)} s | </p>
-//         <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon" onclick="getVideoPlayer().currentTime = '${sponsorBlockTimestamps[1]}'"><p>${sponsorBlockTimestamps[1].toFixed(1)} s | </p>
-//         <img src="/static/img/PlayerUploadIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">Confirm</button>
-//         `;
-//     } else {
-//         sponsorBlockTimestamps.push(currentTime);
-//         sponsorBlockElement.innerHTML = `
-//         <img src="/static/img/PlayerStopIconSponsorBlocker.svg" class="sponsorblockIcon"><button onclick="sendSponsorBlockSegment()">End</button>
-//         `;
-//     }
-// }
-
 // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)`
 function insertVideoTag(videoData, videoProgress) {
    var videoTag = createVideoTag(videoData, videoProgress);
@ -563,23 +513,14 @@ function onVideoProgress() {
    var videoElement = getVideoPlayer();
    // var sponsorBlockElement = document.getElementById("sponsorblock");
    var notificationsElement = document.getElementById("notifications");
-    if (sponsorBlock) {
-        for(let i in sponsorBlock) {
-            if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) {
-                videoElement.currentTime = sponsorBlock[i].segment[1];
-                notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.</h3>`;
+    if (sponsorBlock.segments.length > 0) {
+        for(let i in sponsorBlock.segments) {
+            if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) {
+                videoElement.currentTime = sponsorBlock.segments[i].segment[1];
+                notificationsElement.innerHTML += `<h3 id="notification-${sponsorBlock.segments[i].UUID}">Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.</h3>`;
            }
-            // if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) {
-            //     if(sponsorBlock[i].locked != 1) {
-            //         sponsorBlockElement.innerHTML += `
-            //         <div id="${sponsorBlock[i].UUID}">
-            //             <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', 1)">Up Vote</button>
-            //             <button onclick="sendSponsorBlockVote('${sponsorBlock[i].UUID}', -1)">Down Vote</button>
-            //         </div>`;
-            //     }
-            // }
-            if(currentTime > sponsorBlock[i].segment[1] + 10) {
-                var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID);
+            if(currentTime > sponsorBlock.segments[i].segment[1] + 10) {
+                var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
                if(notificationsElementUUID) {
                    notificationsElementUUID.outerHTML = '';
                }
@ -602,6 +543,12 @@ function onVideoEnded() {
    if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
        updateVideoWatchStatus(videoId, "unwatched");
    }
+    for(let i in sponsorBlock.segments) {
+        var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID);
+        if(notificationsElementUUID) {
+            notificationsElementUUID.outerHTML = '';
+        }
+    }
 }

 function watchedThreshold(currentTime, duration) {