Merge branch 'master' of https://github.com/bbilly1/tubearchivist into feat/react-frontend

README.md

@@ -148,19 +148,20 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex
 - [ ] User roles
 - [ ] Podcast mode to serve channel as mp3
 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage
-- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications (#97)
+- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/bbilly1/tubearchivist/issues/97))
 - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration
-- [ ] User created playlists (#108)
+- [ ] Add passing browser cookies to yt-dlp ([#199](https://github.com/bbilly1/tubearchivist/issues/199))
+- [ ] User created playlists ([#108](https://github.com/bbilly1/tubearchivist/issues/108))
 - [ ] Auto play or play next link
 - [ ] Show similar videos on video page
 - [ ] Multi language support
 - [ ] Show total video downloaded vs total videos available in channel
 - [ ] Make items in grid row configurable to use more of the screen
 - [ ] Add statistics of index
-- [ ] Implement complete offline media file import from json file (#138)
-- [ ] Filter and query in search form, search by url query (#134, #139)
-- [ ] Auto ignore videos by keyword (#163)
-- [ ] Custom searchable notes to videos, channels, playlists (#144)
+- [ ] Implement complete offline media file import from json file ([#138](https://github.com/bbilly1/tubearchivist/issues/138))
+- [ ] Filter and query in search form, search by url query ([#134](https://github.com/bbilly1/tubearchivist/issues/134), [#139](https://github.com/bbilly1/tubearchivist/issues/139))
+- [ ] Auto ignore videos by keyword ([#163](https://github.com/bbilly1/tubearchivist/issues/163))
+- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/bbilly1/tubearchivist/issues/144))
 
 Implemented:
 - [X] Implement per channel settings [2022-03-26]

tubearchivist/api/README.md

@@ -70,6 +70,43 @@ POST /api/video/\<video_id>/progress
 ### Delete player position of video
 DELETE /api/video/\<video_id>/progress  
 
+
+## Sponsor Block View
+/api/video/\<video_id>/sponsor/
+
+Integrate with sponsorblock
+
+### Get list of segments
+GET /api/video/\<video_id>/sponsor/
+
+
+### Vote on existing segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "vote": {
+        "uuid": "<uuid>",
+        "yourVote": 1
+    }
+}
+```
+yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote
+
+### Create new segment
+**This only simulates the request**  
+POST /api/video/\<video_id>/sponsor/
+```json
+{
+    "segment": {
+        "startTime": 5,
+        "endTime": 10
+    }
+}
+```
+Timestamps either *int* or *float*, end time can't be before start time.
+
+
 ## Channel List View
 /api/channel/
 

tubearchivist/api/urls.py

@@ -11,6 +11,7 @@ from api.views import (
     VideoApiListView,
     VideoApiView,
     VideoProgressView,
+    VideoSponsorView,
 )
 from django.urls import path
 
@@ -32,6 +33,11 @@ urlpatterns = [
         VideoProgressView.as_view(),
         name="api-video-progress",
     ),
+    path(
+        "video/<slug:video_id>/sponsor/",
+        VideoSponsorView.as_view(),
+        name="api-video-sponsor",
+    ),
     path(
         "channel/",
         ChannelApiListView.as_view(),

tubearchivist/api/views.py

@@ -3,6 +3,7 @@
 from api.src.search_processor import SearchProcess
 from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
+from home.src.index.video import SponsorBlock
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import UrlListParser
 from home.src.ta.ta_redis import RedisArchivist
@@ -144,6 +145,55 @@ class VideoProgressView(ApiBaseView):
         return Response(self.response)
 
 
+class VideoSponsorView(ApiBaseView):
+    """resolves to /api/video/<video_id>/sponsor/
+    handle sponsor block integration
+    """
+
+    search_base = "ta_video/_doc/"
+
+    def get(self, request, video_id):
+        """get sponsor info"""
+        # pylint: disable=unused-argument
+
+        self.get_document(video_id)
+        sponsorblock = self.response["data"].get("sponsorblock")
+
+        return Response(sponsorblock)
+
+    def post(self, request, video_id):
+        """post verification and timestamps"""
+        if "segment" in request.data:
+            response, status_code = self._create_segment(request, video_id)
+        elif "vote" in request.data:
+            response, status_code = self._vote_on_segment(request)
+
+        return Response(response, status=status_code)
+
+    @staticmethod
+    def _create_segment(request, video_id):
+        """create segment in API"""
+        start_time = request.data["segment"]["startTime"]
+        end_time = request.data["segment"]["endTime"]
+        response, status_code = SponsorBlock(request.user.id).post_timestamps(
+            video_id, start_time, end_time
+        )
+
+        return response, status_code
+
+    @staticmethod
+    def _vote_on_segment(request):
+        """validate on existing segment"""
+        user_id = request.user.id
+        uuid = request.data["vote"]["uuid"]
+        vote = request.data["vote"]["yourVote"]
+        response, status_code = SponsorBlock(user_id).vote_on_segment(
+            uuid, vote
+        )
+
+        return response, status_code
+
+
 class ChannelApiView(ApiBaseView):
     """resolves to /api/channel/<channel_id>/
     GET: returns metadata dict of channel

tubearchivist/config/settings.py

@@ -151,3 +151,7 @@ CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"]
 CORS_ALLOW_HEADERS = list(default_headers) + [
     "mode",
 ]
+
+# TA application settings
+TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist"
+TA_VERSION = "v0.1.3"

tubearchivist/home/config.json

@@ -27,7 +27,8 @@
         "subtitle_source": false,
         "subtitle_index": false,
         "throttledratelimit": false,
-        "integrate_ryd": false
+        "integrate_ryd": false,
+        "integrate_sponsorblock": false
     },
     "application": {
         "app_root": "/app",

tubearchivist/home/src/frontend/forms.py

@@ -62,6 +62,12 @@ class ApplicationSettingsForm(forms.Form):
         ("1", "enable ryd integration"),
     ]
 
+    SP_CHOICES = [
+        ("", "-- change sponsorblock integrations"),
+        ("0", "disable sponsorblock integration"),
+        ("1", "enable sponsorblock integration"),
+    ]
+
     CAST_CHOICES = [
         ("", "-- change Cast integration --"),
         ("0", "disable Cast"),
@@ -103,6 +109,9 @@ class ApplicationSettingsForm(forms.Form):
     downloads_integrate_ryd = forms.ChoiceField(
         widget=forms.Select, choices=RYD_CHOICES, required=False
     )
+    downloads_integrate_sponsorblock = forms.ChoiceField(
+        widget=forms.Select, choices=SP_CHOICES, required=False
+    )
     application_enable_cast = forms.ChoiceField(
         widget=forms.Select, choices=CAST_CHOICES, required=False
     )

tubearchivist/home/src/index/video.py

@@ -9,14 +9,17 @@ import os
 from datetime import datetime
 
 import requests
+from django.conf import settings
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index.generic import YouTubeItem
 from home.src.ta.helper import (
     DurationConverter,
     clean_string,
+    randomizor,
     requests_headers,
 )
+from home.src.ta.ta_redis import RedisArchivist
 from ryd_client import ryd_client
 
 
@@ -280,6 +283,73 @@ class SubtitleParser:
         return chunk_list
 
 
+class SponsorBlock:
+    """handle sponsor block integration"""
+
+    API = "https://sponsor.ajay.app/api"
+
+    def __init__(self, user_id=False):
+        self.user_id = user_id
+        self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}"
+
+    def get_sb_id(self):
+        """get sponsorblock userid or generate if needed"""
+        if not self.user_id:
+            print("missing request user id")
+            raise ValueError
+
+        key = f"{self.user_id}:id_sponsorblock"
+        sb_id = RedisArchivist().get_message(key)
+        if not sb_id["status"]:
+            sb_id = {"status": randomizor(32)}
+            RedisArchivist().set_message(key, sb_id, expire=False)
+
+        return sb_id
+
+    def get_timestamps(self, youtube_id):
+        """get timestamps from the API"""
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        headers = {"User-Agent": self.user_agent}
+        print(f"{youtube_id}: get sponsorblock timestamps")
+        response = requests.get(url, headers=headers)
+        if not response.ok:
+            print(f"{youtube_id}: sponsorblock failed: {response.text}")
+            return False
+
+        return response.json()
+
+    def post_timestamps(self, youtube_id, start_time, end_time):
+        """post timestamps to api"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "videoID": youtube_id,
+            "startTime": start_time,
+            "endTime": end_time,
+            "category": "sponsor",
+            "userID": user_id,
+            "userAgent": self.user_agent,
+        }
+        url = f"{self.API}/skipSegments?videoID={youtube_id}"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+    def vote_on_segment(self, uuid, vote):
+        """send vote on existing segment"""
+        user_id = self.get_sb_id().get("status")
+        data = {
+            "UUID": uuid,
+            "userID": user_id,
+            "type": vote,
+        }
+        url = f"{self.API}/api/voteOnSponsorTime"
+        print(f"post: {data}")
+        print(f"to: {url}")
+
+        return {"success": True}, 200
+
+
 class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
     """represents a single youtube video"""
 
@@ -306,6 +376,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         if self.config["downloads"]["integrate_ryd"]:
             self._get_ryd_stats()
 
+        if self.config["downloads"]["integrate_sponsorblock"]:
+            self._get_sponsorblock()
+
         return
 
     def _process_youtube_meta(self):
@@ -447,6 +520,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
         return True
 
+    def _get_sponsorblock(self):
+        """get optional sponsorblock timestamps from sponsor.ajay.app"""
+        sponsorblock = SponsorBlock().get_timestamps(self.youtube_id)
+        if sponsorblock:
+            self.json_data["sponsorblock"] = sponsorblock
+
     def check_subtitles(self):
         """optionally add subtitles"""
         handler = YoutubeSubtitle(self)

tubearchivist/home/src/ta/helper.py

@@ -37,6 +37,12 @@ def ignore_filelist(filelist):
     return cleaned
 
 
+def randomizor(length):
+    """generate random alpha numeric string"""
+    pool = string.digits + string.ascii_letters
+    return "".join(random.choice(pool) for i in range(length))
+
+
 def requests_headers():
     """build header with random user agent for requests outside of yt-dlp"""
 

tubearchivist/home/templates/home/base.html

@@ -132,7 +132,7 @@
     </div>
     <div class="footer">
         <div class="boxed-content">
-            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist v0.1.3 </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
+            <span>© 2021 - <script type="text/javascript">document.write(new Date().getFullYear());</script> TubeArchivist {{ version }} </span><span><a href="{% url 'about' %}">About</a> | <a href="https://github.com/bbilly1/tubearchivist" target="_blank">GitHub</a> | <a href="https://hub.docker.com/r/bbilly1/tubearchivist" target="_blank">Docker Hub</a> | <a href="https://discord.gg/AFwz8nE7BK" target="_blank">Discord</a> | <a href="https://www.reddit.com/r/TubeArchivist/">Reddit</a></span>
         </div>
     </div>
 </body>

tubearchivist/home/templates/home/settings.html

@@ -128,6 +128,11 @@
                 <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
                 {{ app_form.downloads_integrate_ryd }}
             </div>
+            <div class="settings-item">
+                <p>Integrate with <a href="https://sponsor.ajay.app/">SponsorBlock</a> to get sponsored timestamps: <span class="settings-current">{{ config.downloads.integrate_sponsorblock }}</span></p>
+                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
+                {{ app_form.downloads_integrate_sponsorblock }}
+            </div>
             <div class="settings-item">
                 <p>Current Cast integration: <span class="settings-current">{{ config.application.enable_cast }}</span></p>
                 <i>Enabling Cast will load an additional JS library from Google. HTTPS and a supported browser are required for this integration.</i><br>

tubearchivist/home/views.py

@@ -9,6 +9,7 @@ import urllib.parse
 from time import sleep
 
 from django import forms
+from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
 from django.http import JsonResponse
@@ -122,6 +123,7 @@ class ArchivistViewConfig(View):
             "hide_watched": self._get_hide_watched(),
             "show_ignored_only": self._get_show_ignore_only(),
             "show_subed_only": self._get_show_subed_only(),
+            "version": settings.TA_VERSION,
         }
 
 
@@ -329,8 +331,11 @@ class AboutView(View):
     @staticmethod
     def get(request):
         """handle http get"""
-        colors = AppConfig(request.user.id).colors
-        context = {"title": "About", "colors": colors}
+        context = {
+            "title": "About",
+            "colors": AppConfig(request.user.id).colors,
+            "version": settings.TA_VERSION,
+        }
         return render(request, "home/about.html", context)
 
 
@@ -690,6 +695,7 @@ class VideoView(View):
             "title": video_title,
             "colors": colors,
             "cast": cast,
+            "version": settings.TA_VERSION,
         }
         return render(request, "home/video.html", context)
 
@@ -746,7 +752,10 @@ class SearchView(ArchivistResultsView):
         all_styles = self.get_all_view_styles()
         self.context.update({"all_styles": all_styles})
         self.context.update(
-            {"search_form": MultiSearchForm(initial=all_styles)}
+            {
+                "search_form": MultiSearchForm(initial=all_styles),
+                "version": settings.TA_VERSION,
+            }
         )
 
         return render(request, "home/search.html", self.context)
@@ -778,6 +787,7 @@ class SettingsView(View):
             "user_form": user_form,
             "app_form": app_form,
             "scheduler_form": scheduler_form,
+            "version": settings.TA_VERSION,
         }
 
         return render(request, "home/settings.html", context)

tubearchivist/requirements.txt

@@ -1,10 +1,10 @@
 beautifulsoup4==4.10.0
-celery==5.2.3
+celery==5.2.6
 Django==4.0.3
 django-cors-headers==3.11.0
 djangorestframework==3.13.1
-Pillow==9.0.1
-redis==4.2.1
+Pillow==9.1.0
+redis==4.2.2
 requests==2.27.1
 ryd-client==0.0.3
 uWSGI==2.0.20