diff --git a/README.md b/README.md index 201ed77..91dd601 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage - [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications (#97) - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration +- [ ] Add passing browser cookies to yt-dlp (#199) - [ ] User created playlists (#108) - [ ] Auto play or play next link - [ ] Show similar videos on video page diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 9852208..3c019cd 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -70,6 +70,43 @@ POST /api/video/\/progress ### Delete player position of video DELETE /api/video/\/progress + +## Sponsor Block View +/api/video/\/sponsor/ + +Integrate with sponsorblock + +### Get list of segments +GET /api/video/\/sponsor/ + + +### Vote on existing segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "vote": { + "uuid": "", + "yourVote": 1 + } +} +``` +yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote + +### Create new segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "segment": { + "startTime": 5, + "endTime": 10 + } +} +``` +Timestamps either *int* or *float*, end time can't be before start time. + + ## Channel List View /api/channel/ diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 8c1a5f8..0cdf11a 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -11,6 +11,7 @@ from api.views import ( VideoApiListView, VideoApiView, VideoProgressView, + VideoSponsorView, ) from django.urls import path @@ -32,6 +33,11 @@ urlpatterns = [ VideoProgressView.as_view(), name="api-video-progress", ), + path( + "video//sponsor/", + VideoSponsorView.as_view(), + name="api-video-sponsor", + ), path( "channel/", ChannelApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index bf391a9..d938b2f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,6 +3,7 @@ from api.src.search_processor import SearchProcess from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap +from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist @@ -144,6 +145,55 @@ class VideoProgressView(ApiBaseView): return Response(self.response) +class VideoSponsorView(ApiBaseView): + """resolves to /api/video//sponsor/ + handle sponsor block integration + """ + + search_base = "ta_video/_doc/" + + def get(self, request, video_id): + """get sponsor info""" + # pylint: disable=unused-argument + + self.get_document(video_id) + sponsorblock = self.response["data"].get("sponsorblock") + + return Response(sponsorblock) + + def post(self, request, video_id): + """post verification and timestamps""" + if "segment" in request.data: + response, status_code = self._create_segment(request, video_id) + elif "vote" in request.data: + response, status_code = self._vote_on_segment(request) + + return Response(response, status=status_code) + + @staticmethod + def _create_segment(request, video_id): + """create segment in API""" + start_time = request.data["segment"]["startTime"] + end_time = request.data["segment"]["endTime"] + response, status_code = SponsorBlock(request.user.id).post_timestamps( + video_id, start_time, end_time + ) + + return response, status_code + + @staticmethod + def _vote_on_segment(request): + """validate on existing segment""" + user_id = request.user.id + uuid = request.data["vote"]["uuid"] + vote = request.data["vote"]["yourVote"] + response, status_code = SponsorBlock(user_id).vote_on_segment( + uuid, vote + ) + + return response, status_code + + class ChannelApiView(ApiBaseView): """resolves to /api/channel// GET: returns metadata dict of channel diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 5447788..251ad5c 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -151,3 +151,7 @@ CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"] CORS_ALLOW_HEADERS = list(default_headers) + [ "mode", ] + +# TA application settings +TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist" +TA_VERSION = "v0.1.3" diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 559ad2c..411b4af 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -9,14 +9,17 @@ import os from datetime import datetime import requests +from django.conf import settings from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem from home.src.ta.helper import ( DurationConverter, clean_string, + randomizor, requests_headers, ) +from home.src.ta.ta_redis import RedisArchivist from ryd_client import ryd_client @@ -280,6 +283,73 @@ class SubtitleParser: return chunk_list +class SponsorBlock: + """handle sponsor block integration""" + + API = "https://sponsor.ajay.app/api" + + def __init__(self, user_id=False): + self.user_id = user_id + self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" + + def get_sb_id(self): + """get sponsorblock userid or generate if needed""" + if not self.user_id: + print("missing request user id") + raise ValueError + + key = f"{self.user_id}:id_sponsorblock" + sb_id = RedisArchivist().get_message(key) + if not sb_id["status"]: + sb_id = {"status": randomizor(32)} + RedisArchivist().set_message(key, sb_id, expire=False) + + return sb_id + + def get_timestamps(self, youtube_id): + """get timestamps from the API""" + url = f"{self.API}/skipSegments?videoID={youtube_id}" + headers = {"User-Agent": self.user_agent} + print(f"{youtube_id}: get sponsorblock timestamps") + response = requests.get(url, headers=headers) + if not response.ok: + print(f"{youtube_id}: sponsorblock failed: {response.text}") + return False + + return response.json() + + def post_timestamps(self, youtube_id, start_time, end_time): + """post timestamps to api""" + user_id = self.get_sb_id().get("status") + data = { + "videoID": youtube_id, + "startTime": start_time, + "endTime": end_time, + "category": "sponsor", + "userID": user_id, + "userAgent": self.user_agent, + } + url = f"{self.API}/skipSegments?videoID={youtube_id}" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + + def vote_on_segment(self, uuid, vote): + """send vote on existing segment""" + user_id = self.get_sb_id().get("status") + data = { + "UUID": uuid, + "userID": user_id, + "type": vote, + } + url = f"{self.API}/api/voteOnSponsorTime" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + + class YoutubeVideo(YouTubeItem, YoutubeSubtitle): """represents a single youtube video""" @@ -452,15 +522,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _get_sponsorblock(self): """get optional sponsorblock timestamps from sponsor.ajay.app""" - api = "https://sponsor.ajay.app/api" - url = f"{api}/skipSegments?videoID={self.youtube_id}" - print(f"{self.youtube_id}: get sponsorblock timestamps") - response = requests.get(url) - if not response.ok: - print(f"{self.youtube_id}: sponsorblock failed: {response.text}") - return - - self.json_data["sponsorblock"] = response.json() + sponsorblock = SponsorBlock().get_timestamps(self.youtube_id) + if sponsorblock: + self.json_data["sponsorblock"] = sponsorblock def check_subtitles(self): """optionally add subtitles""" diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 31ba727..c572ccc 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -37,6 +37,12 @@ def ignore_filelist(filelist): return cleaned +def randomizor(length): + """generate random alpha numeric string""" + pool = string.digits + string.ascii_letters + return "".join(random.choice(pool) for i in range(length)) + + def requests_headers(): """build header with random user agent for requests outside of yt-dlp""" diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index f2a1765..ceb35d5 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -132,7 +132,7 @@
- © 2021 - TubeArchivist v0.1.3 About | GitHub | Docker Hub | Discord | Reddit + © 2021 - TubeArchivist {{ version }} About | GitHub | Docker Hub | Discord | Reddit
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index d8c34ea..19d1cfe 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,6 +9,7 @@ import urllib.parse from time import sleep from django import forms +from django.conf import settings from django.contrib.auth import login from django.contrib.auth.forms import AuthenticationForm from django.http import JsonResponse @@ -122,6 +123,7 @@ class ArchivistViewConfig(View): "hide_watched": self._get_hide_watched(), "show_ignored_only": self._get_show_ignore_only(), "show_subed_only": self._get_show_subed_only(), + "version": settings.TA_VERSION, } @@ -329,8 +331,11 @@ class AboutView(View): @staticmethod def get(request): """handle http get""" - colors = AppConfig(request.user.id).colors - context = {"title": "About", "colors": colors} + context = { + "title": "About", + "colors": AppConfig(request.user.id).colors, + "version": settings.TA_VERSION, + } return render(request, "home/about.html", context) @@ -690,6 +695,7 @@ class VideoView(View): "title": video_title, "colors": colors, "cast": cast, + "version": settings.TA_VERSION, } return render(request, "home/video.html", context) @@ -746,7 +752,10 @@ class SearchView(ArchivistResultsView): all_styles = self.get_all_view_styles() self.context.update({"all_styles": all_styles}) self.context.update( - {"search_form": MultiSearchForm(initial=all_styles)} + { + "search_form": MultiSearchForm(initial=all_styles), + "version": settings.TA_VERSION, + } ) return render(request, "home/search.html", self.context) @@ -778,6 +787,7 @@ class SettingsView(View): "user_form": user_form, "app_form": app_form, "scheduler_form": scheduler_form, + "version": settings.TA_VERSION, } return render(request, "home/settings.html", context) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index b6ad2a8..af75bb3 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4==4.10.0 -celery==5.2.5 +celery==5.2.6 Django==4.0.3 django-cors-headers==3.11.0 djangorestframework==3.13.1