simulate sponsorblock voting api endpoints, #build

Changed:
- added simulated post sponsorblock api endpoints
- updated api documentation
- updated roadmap

README.md

@@ -150,6 +150,7 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
 - [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications (#97)
 - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration
+- [ ] Add passing browser cookies to yt-dlp (#199)
 - [ ] User created playlists (#108)
 - [ ] Auto play or play next link
 - [ ] Show similar videos on video page

tubearchivist/api/README.md

@@ -70,6 +70,43 @@ POST /api/video/\<video_id>/progress
 ### Delete player position of video
 DELETE /api/video/\<video_id>/progress  
 
+
+## Sponsor Block View
+/api/video/\<video_id>/sponsor/
+
+Integrate with sponsorblock
+
+### Get list of segments
+GET /api/video/\<video_id>/sponsor/
+
+
+### Vote on existing segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "vote": {
+        "uuid": "<uuid>",
+        "yourVote": 1
+    }
+}
+```
+yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote
+
+### Create new segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "segment": {
+        "startTime": 5,
+        "endTime": 10
+    }
+}
+```
+Timestamps either *int* or *float*, end time can't be before start time.
+
+
 ## Channel List View
 /api/channel/
 

tubearchivist/api/urls.py

@@ -11,6 +11,7 @@ from api.views import (
     VideoApiListView,
     VideoApiView,
     VideoProgressView,
+    VideoSponsorView,
 )
 from django.urls import path
 
@@ -32,6 +33,11 @@ urlpatterns = [
         VideoProgressView.as_view(),
         name="api-video-progress",
     ),
+    path(
+        "video/<slug:video_id>/sponsor/",
+        VideoSponsorView.as_view(),
+        name="api-video-sponsor",
+    ),
     path(
         "channel/",
         ChannelApiListView.as_view(),

tubearchivist/api/views.py

@@ -3,6 +3,7 @@
 from api.src.search_processor import SearchProcess
 from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
+from home.src.index.video import SponsorBlock
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist
@@ -144,6 +145,55 @@ class VideoProgressView(ApiBaseView):
         return Response(self.response)
 
 
+class VideoSponsorView(ApiBaseView):
+    """resolves to /api/video/<video_id>/sponsor/
+    handle sponsor block integration
+    """
+
+    search_base = "ta_video/_doc/"
+
+    def get(self, request, video_id):
+        """get sponsor info"""
+        # pylint: disable=unused-argument
+
+        self.get_document(video_id)
+        sponsorblock = self.response["data"].get("sponsorblock")
+
+        return Response(sponsorblock)
+
+    def post(self, request, video_id):
+        """post verification and timestamps"""
+        if "segment" in request.data:
+            response, status_code = self._create_segment(request, video_id)
+        elif "vote" in request.data:
+            response, status_code = self._vote_on_segment(request)
+
+        return Response(response, status=status_code)
+
+    @staticmethod
+    def _create_segment(request, video_id):
+        """create segment in API"""
+        start_time = request.data["segment"]["startTime"]
+        end_time = request.data["segment"]["endTime"]
+        response, status_code = SponsorBlock(request.user.id).post_timestamps(
+            video_id, start_time, end_time
+        )
+
+        return response, status_code
+
+    @staticmethod
+    def _vote_on_segment(request):
+        """validate on existing segment"""
+        user_id = request.user.id
+        uuid = request.data["vote"]["uuid"]
+        vote = request.data["vote"]["yourVote"]
+        response, status_code = SponsorBlock(user_id).vote_on_segment(
+            uuid, vote
+        )
+
+        return response, status_code
+
+
 class ChannelApiView(ApiBaseView):
     """resolves to /api/channel/<channel_id>/
     GET: returns metadata dict of channel

tubearchivist/config/settings.py

@@ -151,3 +151,7 @@ CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"]
 CORS_ALLOW_HEADERS = list(default_headers) + [
     "mode",
 ]
+
+# TA application settings
+TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist"
+TA_VERSION = "v0.1.3"

tubearchivist/home/src/index/video.py

@@ -9,14 +9,17 @@ import os
 from datetime import datetime
 
 import requests
+from django.conf import settings
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index.generic import YouTubeItem
 from home.src.ta.helper import (
     DurationConverter,
     clean_string,
+    randomizor,
     requests_headers,
 )
+from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
 
@@ -280,6 +283,73 @@ class SubtitleParser:
         return chunk_list
 
 
+class SponsorBlock:
+    """handle sponsor block integration"""
+
+    API = "https://sponsor.ajay.app/api"
+
+    def __init__(self, user_id=False):
+        self.user_id = user_id
+        self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+
+    def get_sb_id(self):
+        """get sponsorblock userid or generate if needed"""
+        if not self.user_id:
+            print("missing request user id")
+            raise ValueError
+
+        key = f"{self.user_id}:id_sponsorblock"
+        sb_id = RedisArchivist().get_message(key)
+        if not sb_id["status"]:
+            sb_id = {"status": randomizor(32)}
+            RedisArchivist().set_message(key, sb_id, expire=False)
+
+        return sb_id
+
+    def get_timestamps(self, youtube_id):
+        """get timestamps from the API"""
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        headers = {"User-Agent": self.user_agent}
+        print(f"{youtube_id}: get sponsorblock timestamps")
+        response = requests.get(url, headers=headers)
+        if not response.ok:
+            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            return False
+
+        return response.json()
+
+    def post_timestamps(self, youtube_id, start_time, end_time):
+        """post timestamps to api"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "videoID": youtube_id,
+            "startTime": start_time,
+            "endTime": end_time,
+            "category": "sponsor",
+            "userID": user_id,
+            "userAgent": self.user_agent,
+        }
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+    def vote_on_segment(self, uuid, vote):
+        """send vote on existing segment"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "UUID": uuid,
+            "userID": user_id,
+            "type": vote,
+        }
+        url = f"{self.API}/api/voteOnSponsorTime"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+
 class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     """represents a single youtube video"""
 
@@ -452,15 +522,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def _get_sponsorblock(self):
         """get optional sponsorblock timestamps from sponsor.ajay.app"""
-        api = "https://sponsor.ajay.app/api"
+        sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
-        url = f"{api}/skipSegments?videoID={self.youtube_id}"
+        if sponsorblock:
-        print(f"{self.youtube_id}: get sponsorblock timestamps")
+            self.json_data["sponsorblock"] = sponsorblock
-        response = requests.get(url)
-        if not response.ok:
-            print(f"{self.youtube_id}: sponsorblock failed: {response.text}")
-            return
-
-        self.json_data["sponsorblock"] = response.json()
 
     def check_subtitles(self):
         """optionally add subtitles"""

tubearchivist/home/src/ta/helper.py

@@ -37,6 +37,12 @@ def ignore_filelist(filelist):
     return cleaned
 
 
+def randomizor(length):
+    """generate random alpha numeric string"""
+    pool = string.digits + string.ascii_letters
+    return "".join(random.choice(pool) for i in range(length))
+
+
 def requests_headers():
     """build header with random user agent for requests outside of yt-dlp"""
 

tubearchivist/home/templates/home/base.html

@@ -132,7 +132,7 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.3 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist {{ version }} </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/views.py

@@ -9,6 +9,7 @@ import urllib.parse
 from time import sleep
 
 from django import forms
+from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
 from django.http import JsonResponse
@@ -122,6 +123,7 @@ class ArchivistViewConfig(View):
             "hide_watched": self._get_hide_watched(),
             "show_ignored_only": self._get_show_ignore_only(),
             "show_subed_only": self._get_show_subed_only(),
+            "version": settings.TA_VERSION,
         }
 
 
@@ -329,8 +331,11 @@ class AboutView(View):
     @staticmethod
     def get(request):
         """handle http get"""
-        colors = AppConfig(request.user.id).colors
+        context = {
-        context = {"title": "About", "colors": colors}
+            "title": "About",
+            "colors": AppConfig(request.user.id).colors,
+            "version": settings.TA_VERSION,
+        }
         return render(request, "home/about.html", context)
 
 
@@ -690,6 +695,7 @@ class VideoView(View):
             "title": video_title,
             "colors": colors,
             "cast": cast,
+            "version": settings.TA_VERSION,
         }
         return render(request, "home/video.html", context)
 
@@ -746,7 +752,10 @@ class SearchView(ArchivistResultsView):
         all_styles = self.get_all_view_styles()
         self.context.update({"all_styles": all_styles})
         self.context.update(
-            {"search_form": MultiSearchForm(initial=all_styles)}
+            {
+                "search_form": MultiSearchForm(initial=all_styles),
+                "version": settings.TA_VERSION,
+            }
         )
 
         return render(request, "home/search.html", self.context)
@@ -778,6 +787,7 @@ class SettingsView(View):
             "user_form": user_form,
             "app_form": app_form,
             "scheduler_form": scheduler_form,
+            "version": settings.TA_VERSION,
         }
 
         return render(request, "home/settings.html", context)

tubearchivist/requirements.txt

@@ -1,5 +1,5 @@
 beautifulsoup4==4.10.0
-celery==5.2.5
+celery==5.2.6
 Django==4.0.3
 django-cors-headers==3.11.0
 djangorestframework==3.13.1