From e404bac7c3b9d5790b0cd49bbc7d21fdf0f8b4a6 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 18:35:06 +0700 Subject: [PATCH 1/8] bump celery --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index b6ad2a8..af75bb3 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4==4.10.0 -celery==5.2.5 +celery==5.2.6 Django==4.0.3 django-cors-headers==3.11.0 djangorestframework==3.13.1 From 1ac92254adaa2c7cacfbb928fd6da2b099e42b02 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 18:43:15 +0700 Subject: [PATCH 2/8] add global vars for version and url --- tubearchivist/config/settings.py | 4 ++++ tubearchivist/home/templates/home/base.html | 2 +- tubearchivist/home/views.py | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 5447788..251ad5c 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -151,3 +151,7 @@ CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"] CORS_ALLOW_HEADERS = list(default_headers) + [ "mode", ] + +# TA application settings +TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist" +TA_VERSION = "v0.1.3" diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index f2a1765..ceb35d5 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -132,7 +132,7 @@
- © 2021 - TubeArchivist v0.1.3 About | GitHub | Docker Hub | Discord | Reddit + © 2021 - TubeArchivist {{ version }} About | GitHub | Docker Hub | Discord | Reddit
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index d8c34ea..19d1cfe 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,6 +9,7 @@ import urllib.parse from time import sleep from django import forms +from django.conf import settings from django.contrib.auth import login from django.contrib.auth.forms import AuthenticationForm from django.http import JsonResponse @@ -122,6 +123,7 @@ class ArchivistViewConfig(View): "hide_watched": self._get_hide_watched(), "show_ignored_only": self._get_show_ignore_only(), "show_subed_only": self._get_show_subed_only(), + "version": settings.TA_VERSION, } @@ -329,8 +331,11 @@ class AboutView(View): @staticmethod def get(request): """handle http get""" - colors = AppConfig(request.user.id).colors - context = {"title": "About", "colors": colors} + context = { + "title": "About", + "colors": AppConfig(request.user.id).colors, + "version": settings.TA_VERSION, + } return render(request, "home/about.html", context) @@ -690,6 +695,7 @@ class VideoView(View): "title": video_title, "colors": colors, "cast": cast, + "version": settings.TA_VERSION, } return render(request, "home/video.html", context) @@ -746,7 +752,10 @@ class SearchView(ArchivistResultsView): all_styles = self.get_all_view_styles() self.context.update({"all_styles": all_styles}) self.context.update( - {"search_form": MultiSearchForm(initial=all_styles)} + { + "search_form": MultiSearchForm(initial=all_styles), + "version": settings.TA_VERSION, + } ) return render(request, "home/search.html", self.context) @@ -778,6 +787,7 @@ class SettingsView(View): "user_form": user_form, "app_form": app_form, "scheduler_form": scheduler_form, + "version": settings.TA_VERSION, } return render(request, "home/settings.html", context) From 738b083a7fbc686b8aa124f14fff41097cc52ced Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 21:50:40 +0700 Subject: [PATCH 3/8] create dedicated SponsorBlock class to handle integration --- tubearchivist/home/src/index/video.py | 68 +++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 559ad2c..108a3b7 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -9,14 +9,17 @@ import os from datetime import datetime import requests +from django.conf import settings from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem from home.src.ta.helper import ( DurationConverter, clean_string, + randomizor, requests_headers, ) +from home.src.ta.ta_redis import RedisArchivist from ryd_client import ryd_client @@ -280,6 +283,59 @@ class SubtitleParser: return chunk_list +class SponsorBlock: + """handle sponsor block integration""" + + API = "https://sponsor.ajay.app/api" + + def __init__(self, user_id=False): + self.user_id = user_id + self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" + + def get_sb_id(self): + """get sponsorblock userid or generate if needed""" + if not self.user_id: + print("missing request user id") + raise ValueError + + key = f"{self.user_id}:id_sponsorblock" + sb_id = RedisArchivist().get_message(key) + if not sb_id["status"]: + sb_id = {"status": randomizor(32)} + RedisArchivist().set_message(key, sb_id, expire=False) + + return sb_id + + def get_timestamps(self, youtube_id): + """get timestamps from the API""" + url = f"{self.API}/skipSegments?videoID={youtube_id}" + headers = {"User-Agent": self.user_agent} + print(f"{youtube_id}: get sponsorblock timestamps") + response = requests.get(url, headers=headers) + if not response.ok: + print(f"{youtube_id}: sponsorblock failed: {response.text}") + return False + + return response.json() + + def post_timestamps(self, youtube_id, start_time, end_time): + """post timestamps to api""" + user_id = self.get_sb_id().get("status") + data = { + "videoID": youtube_id, + "startTime": start_time, + "endTime": end_time, + "category": "sponsor", + "userID": user_id, + "userAgent": self.user_agent, + } + url = f"{self.API}/skipSegments?videoID={youtube_id}" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + + class YoutubeVideo(YouTubeItem, YoutubeSubtitle): """represents a single youtube video""" @@ -452,15 +508,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _get_sponsorblock(self): """get optional sponsorblock timestamps from sponsor.ajay.app""" - api = "https://sponsor.ajay.app/api" - url = f"{api}/skipSegments?videoID={self.youtube_id}" - print(f"{self.youtube_id}: get sponsorblock timestamps") - response = requests.get(url) - if not response.ok: - print(f"{self.youtube_id}: sponsorblock failed: {response.text}") - return - - self.json_data["sponsorblock"] = response.json() + sponsorblock = SponsorBlock().get_timestamps(self.youtube_id) + if sponsorblock: + self.json_data["sponsorblock"] = sponsorblock def check_subtitles(self): """optionally add subtitles""" From bace7d41af3777580fc82e0bb20a67db702a79ce Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 21:51:10 +0700 Subject: [PATCH 4/8] add random string generator helper function --- tubearchivist/home/src/ta/helper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 31ba727..c572ccc 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -37,6 +37,12 @@ def ignore_filelist(filelist): return cleaned +def randomizor(length): + """generate random alpha numeric string""" + pool = string.digits + string.ascii_letters + return "".join(random.choice(pool) for i in range(length)) + + def requests_headers(): """build header with random user agent for requests outside of yt-dlp""" From 286b1cf9b67f2ff0080a38cd18f2286b03624e4b Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 21:51:45 +0700 Subject: [PATCH 5/8] simulate sponsorblock post request --- tubearchivist/api/urls.py | 6 ++++++ tubearchivist/api/views.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 8c1a5f8..0cdf11a 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -11,6 +11,7 @@ from api.views import ( VideoApiListView, VideoApiView, VideoProgressView, + VideoSponsorView, ) from django.urls import path @@ -32,6 +33,11 @@ urlpatterns = [ VideoProgressView.as_view(), name="api-video-progress", ), + path( + "video//sponsor/", + VideoSponsorView.as_view(), + name="api-video-sponsor", + ), path( "channel/", ChannelApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index bf391a9..f590d43 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,6 +3,7 @@ from api.src.search_processor import SearchProcess from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap +from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist @@ -144,6 +145,35 @@ class VideoProgressView(ApiBaseView): return Response(self.response) +class VideoSponsorView(ApiBaseView): + """resolves to /api/video// + handle sponsor block integration + """ + + search_base = "ta_video/_doc/" + + def get(self, request, video_id): + """get sponsor info""" + # pylint: disable=unused-argument + + self.get_document(video_id) + sponsorblock = self.response["data"].get("sponsorblock") + + return Response(sponsorblock) + + @staticmethod + def post(request, video_id): + """post verification and timestamps""" + start_time = request.data.get("startTime") + end_time = request.data.get("endTime") + + response, status_code = SponsorBlock(request.user.id).post_timestamps( + video_id, start_time, end_time + ) + + return Response(response, status=status_code) + + class ChannelApiView(ApiBaseView): """resolves to /api/channel// GET: returns metadata dict of channel From 03dd25cff4a589a2d76ebd912a7815b09d001af8 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 22:25:40 +0700 Subject: [PATCH 6/8] implement vote on sponsorblock segments api --- tubearchivist/api/views.py | 32 ++++++++++++++++++++++----- tubearchivist/home/src/index/video.py | 14 ++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index f590d43..d938b2f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -146,7 +146,7 @@ class VideoProgressView(ApiBaseView): class VideoSponsorView(ApiBaseView): - """resolves to /api/video// + """resolves to /api/video//sponsor/ handle sponsor block integration """ @@ -161,17 +161,37 @@ class VideoSponsorView(ApiBaseView): return Response(sponsorblock) - @staticmethod - def post(request, video_id): + def post(self, request, video_id): """post verification and timestamps""" - start_time = request.data.get("startTime") - end_time = request.data.get("endTime") + if "segment" in request.data: + response, status_code = self._create_segment(request, video_id) + elif "vote" in request.data: + response, status_code = self._vote_on_segment(request) + return Response(response, status=status_code) + + @staticmethod + def _create_segment(request, video_id): + """create segment in API""" + start_time = request.data["segment"]["startTime"] + end_time = request.data["segment"]["endTime"] response, status_code = SponsorBlock(request.user.id).post_timestamps( video_id, start_time, end_time ) - return Response(response, status=status_code) + return response, status_code + + @staticmethod + def _vote_on_segment(request): + """validate on existing segment""" + user_id = request.user.id + uuid = request.data["vote"]["uuid"] + vote = request.data["vote"]["yourVote"] + response, status_code = SponsorBlock(user_id).vote_on_segment( + uuid, vote + ) + + return response, status_code class ChannelApiView(ApiBaseView): diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 108a3b7..411b4af 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -335,6 +335,20 @@ class SponsorBlock: return {"success": True}, 200 + def vote_on_segment(self, uuid, vote): + """send vote on existing segment""" + user_id = self.get_sb_id().get("status") + data = { + "UUID": uuid, + "userID": user_id, + "type": vote, + } + url = f"{self.API}/api/voteOnSponsorTime" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + class YoutubeVideo(YouTubeItem, YoutubeSubtitle): """represents a single youtube video""" From a84e6575778fd25a346347811c84b583aff480eb Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 22:27:55 +0700 Subject: [PATCH 7/8] add documentation for sponsorblock api endpoints --- tubearchivist/api/README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 9852208..3c019cd 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -70,6 +70,43 @@ POST /api/video/\/progress ### Delete player position of video DELETE /api/video/\/progress + +## Sponsor Block View +/api/video/\/sponsor/ + +Integrate with sponsorblock + +### Get list of segments +GET /api/video/\/sponsor/ + + +### Vote on existing segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "vote": { + "uuid": "", + "yourVote": 1 + } +} +``` +yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote + +### Create new segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "segment": { + "startTime": 5, + "endTime": 10 + } +} +``` +Timestamps either *int* or *float*, end time can't be before start time. + + ## Channel List View /api/channel/ From 348a114981e0b8907015d5c276de3edb2d64c3b8 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 5 Apr 2022 22:32:55 +0700 Subject: [PATCH 8/8] update Roadmap --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 201ed77..91dd601 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage - [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications (#97) - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration +- [ ] Add passing browser cookies to yt-dlp (#199) - [ ] User created playlists (#108) - [ ] Auto play or play next link - [ ] Show similar videos on video page