diff --git a/README.md b/README.md index 201ed77..9d4b259 100644 --- a/README.md +++ b/README.md @@ -148,19 +148,20 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] User roles - [ ] Podcast mode to serve channel as mp3 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage -- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications (#97) +- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/bbilly1/tubearchivist/issues/97)) - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration -- [ ] User created playlists (#108) +- [ ] Add passing browser cookies to yt-dlp ([#199](https://github.com/bbilly1/tubearchivist/issues/199)) +- [ ] User created playlists ([#108](https://github.com/bbilly1/tubearchivist/issues/108)) - [ ] Auto play or play next link - [ ] Show similar videos on video page - [ ] Multi language support - [ ] Show total video downloaded vs total videos available in channel - [ ] Make items in grid row configurable to use more of the screen - [ ] Add statistics of index -- [ ] Implement complete offline media file import from json file (#138) -- [ ] Filter and query in search form, search by url query (#134, #139) -- [ ] Auto ignore videos by keyword (#163) -- [ ] Custom searchable notes to videos, channels, playlists (#144) +- [ ] Implement complete offline media file import from json file ([#138](https://github.com/bbilly1/tubearchivist/issues/138)) +- [ ] Filter and query in search form, search by url query ([#134](https://github.com/bbilly1/tubearchivist/issues/134), [#139](https://github.com/bbilly1/tubearchivist/issues/139)) +- [ ] Auto ignore videos by keyword ([#163](https://github.com/bbilly1/tubearchivist/issues/163)) +- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/bbilly1/tubearchivist/issues/144)) Implemented: - [X] Implement per channel settings [2022-03-26] diff --git a/docs/assets/TubeArchivist-ES.png b/docs/assets/TubeArchivist-ES.png new file mode 100644 index 0000000..ae8f656 Binary files /dev/null and b/docs/assets/TubeArchivist-ES.png differ diff --git a/docs/assets/TubeArchivist-RedisJSON.png b/docs/assets/TubeArchivist-RedisJSON.png new file mode 100644 index 0000000..c86c37a Binary files /dev/null and b/docs/assets/TubeArchivist-RedisJSON.png differ diff --git a/docs/assets/TubeArchivist.png b/docs/assets/TubeArchivist.png new file mode 100644 index 0000000..20c61d7 Binary files /dev/null and b/docs/assets/TubeArchivist.png differ diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index 9852208..3c019cd 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -70,6 +70,43 @@ POST /api/video/\/progress ### Delete player position of video DELETE /api/video/\/progress + +## Sponsor Block View +/api/video/\/sponsor/ + +Integrate with sponsorblock + +### Get list of segments +GET /api/video/\/sponsor/ + + +### Vote on existing segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "vote": { + "uuid": "", + "yourVote": 1 + } +} +``` +yourVote needs to be *int*: 0 for downvote, 1 for upvote, 20 to undo vote + +### Create new segment +**This only simulates the request** +POST /api/video/\/sponsor/ +```json +{ + "segment": { + "startTime": 5, + "endTime": 10 + } +} +``` +Timestamps either *int* or *float*, end time can't be before start time. + + ## Channel List View /api/channel/ diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 8c1a5f8..0cdf11a 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -11,6 +11,7 @@ from api.views import ( VideoApiListView, VideoApiView, VideoProgressView, + VideoSponsorView, ) from django.urls import path @@ -32,6 +33,11 @@ urlpatterns = [ VideoProgressView.as_view(), name="api-video-progress", ), + path( + "video//sponsor/", + VideoSponsorView.as_view(), + name="api-video-sponsor", + ), path( "channel/", ChannelApiListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index bf391a9..d938b2f 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,6 +3,7 @@ from api.src.search_processor import SearchProcess from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap +from home.src.index.video import SponsorBlock from home.src.ta.config import AppConfig from home.src.ta.helper import UrlListParser from home.src.ta.ta_redis import RedisArchivist @@ -144,6 +145,55 @@ class VideoProgressView(ApiBaseView): return Response(self.response) +class VideoSponsorView(ApiBaseView): + """resolves to /api/video//sponsor/ + handle sponsor block integration + """ + + search_base = "ta_video/_doc/" + + def get(self, request, video_id): + """get sponsor info""" + # pylint: disable=unused-argument + + self.get_document(video_id) + sponsorblock = self.response["data"].get("sponsorblock") + + return Response(sponsorblock) + + def post(self, request, video_id): + """post verification and timestamps""" + if "segment" in request.data: + response, status_code = self._create_segment(request, video_id) + elif "vote" in request.data: + response, status_code = self._vote_on_segment(request) + + return Response(response, status=status_code) + + @staticmethod + def _create_segment(request, video_id): + """create segment in API""" + start_time = request.data["segment"]["startTime"] + end_time = request.data["segment"]["endTime"] + response, status_code = SponsorBlock(request.user.id).post_timestamps( + video_id, start_time, end_time + ) + + return response, status_code + + @staticmethod + def _vote_on_segment(request): + """validate on existing segment""" + user_id = request.user.id + uuid = request.data["vote"]["uuid"] + vote = request.data["vote"]["yourVote"] + response, status_code = SponsorBlock(user_id).vote_on_segment( + uuid, vote + ) + + return response, status_code + + class ChannelApiView(ApiBaseView): """resolves to /api/channel// GET: returns metadata dict of channel diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 5447788..251ad5c 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -151,3 +151,7 @@ CORS_ALLOWED_ORIGIN_REGEXES = [r"moz-extension://*", r"chrome-extension://*"] CORS_ALLOW_HEADERS = list(default_headers) + [ "mode", ] + +# TA application settings +TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist" +TA_VERSION = "v0.1.3" diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index 1b6000e..edb6356 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -27,7 +27,8 @@ "subtitle_source": false, "subtitle_index": false, "throttledratelimit": false, - "integrate_ryd": false + "integrate_ryd": false, + "integrate_sponsorblock": false }, "application": { "app_root": "/app", diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index c4ac817..53ef284 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -62,6 +62,12 @@ class ApplicationSettingsForm(forms.Form): ("1", "enable ryd integration"), ] + SP_CHOICES = [ + ("", "-- change sponsorblock integrations"), + ("0", "disable sponsorblock integration"), + ("1", "enable sponsorblock integration"), + ] + CAST_CHOICES = [ ("", "-- change Cast integration --"), ("0", "disable Cast"), @@ -103,6 +109,9 @@ class ApplicationSettingsForm(forms.Form): downloads_integrate_ryd = forms.ChoiceField( widget=forms.Select, choices=RYD_CHOICES, required=False ) + downloads_integrate_sponsorblock = forms.ChoiceField( + widget=forms.Select, choices=SP_CHOICES, required=False + ) application_enable_cast = forms.ChoiceField( widget=forms.Select, choices=CAST_CHOICES, required=False ) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 1dafb7b..411b4af 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -9,14 +9,17 @@ import os from datetime import datetime import requests +from django.conf import settings from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem from home.src.ta.helper import ( DurationConverter, clean_string, + randomizor, requests_headers, ) +from home.src.ta.ta_redis import RedisArchivist from ryd_client import ryd_client @@ -280,6 +283,73 @@ class SubtitleParser: return chunk_list +class SponsorBlock: + """handle sponsor block integration""" + + API = "https://sponsor.ajay.app/api" + + def __init__(self, user_id=False): + self.user_id = user_id + self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" + + def get_sb_id(self): + """get sponsorblock userid or generate if needed""" + if not self.user_id: + print("missing request user id") + raise ValueError + + key = f"{self.user_id}:id_sponsorblock" + sb_id = RedisArchivist().get_message(key) + if not sb_id["status"]: + sb_id = {"status": randomizor(32)} + RedisArchivist().set_message(key, sb_id, expire=False) + + return sb_id + + def get_timestamps(self, youtube_id): + """get timestamps from the API""" + url = f"{self.API}/skipSegments?videoID={youtube_id}" + headers = {"User-Agent": self.user_agent} + print(f"{youtube_id}: get sponsorblock timestamps") + response = requests.get(url, headers=headers) + if not response.ok: + print(f"{youtube_id}: sponsorblock failed: {response.text}") + return False + + return response.json() + + def post_timestamps(self, youtube_id, start_time, end_time): + """post timestamps to api""" + user_id = self.get_sb_id().get("status") + data = { + "videoID": youtube_id, + "startTime": start_time, + "endTime": end_time, + "category": "sponsor", + "userID": user_id, + "userAgent": self.user_agent, + } + url = f"{self.API}/skipSegments?videoID={youtube_id}" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + + def vote_on_segment(self, uuid, vote): + """send vote on existing segment""" + user_id = self.get_sb_id().get("status") + data = { + "UUID": uuid, + "userID": user_id, + "type": vote, + } + url = f"{self.API}/api/voteOnSponsorTime" + print(f"post: {data}") + print(f"to: {url}") + + return {"success": True}, 200 + + class YoutubeVideo(YouTubeItem, YoutubeSubtitle): """represents a single youtube video""" @@ -306,6 +376,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if self.config["downloads"]["integrate_ryd"]: self._get_ryd_stats() + if self.config["downloads"]["integrate_sponsorblock"]: + self._get_sponsorblock() + return def _process_youtube_meta(self): @@ -447,6 +520,12 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): return True + def _get_sponsorblock(self): + """get optional sponsorblock timestamps from sponsor.ajay.app""" + sponsorblock = SponsorBlock().get_timestamps(self.youtube_id) + if sponsorblock: + self.json_data["sponsorblock"] = sponsorblock + def check_subtitles(self): """optionally add subtitles""" handler = YoutubeSubtitle(self) diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 31ba727..c572ccc 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -37,6 +37,12 @@ def ignore_filelist(filelist): return cleaned +def randomizor(length): + """generate random alpha numeric string""" + pool = string.digits + string.ascii_letters + return "".join(random.choice(pool) for i in range(length)) + + def requests_headers(): """build header with random user agent for requests outside of yt-dlp""" diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index f2a1765..ceb35d5 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -132,7 +132,7 @@
- © 2021 - TubeArchivist v0.1.3 About | GitHub | Docker Hub | Discord | Reddit + © 2021 - TubeArchivist {{ version }} About | GitHub | Docker Hub | Discord | Reddit
diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 740ca40..e4d9534 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -128,6 +128,11 @@ Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_ryd }} +
+

Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}

+ Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
+ {{ app_form.downloads_integrate_sponsorblock }} +

Current Cast integration: {{ config.application.enable_cast }}

Enabling Cast will load an additional JS library from Google. HTTPS and a supported browser are required for this integration.
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index d8c34ea..19d1cfe 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,6 +9,7 @@ import urllib.parse from time import sleep from django import forms +from django.conf import settings from django.contrib.auth import login from django.contrib.auth.forms import AuthenticationForm from django.http import JsonResponse @@ -122,6 +123,7 @@ class ArchivistViewConfig(View): "hide_watched": self._get_hide_watched(), "show_ignored_only": self._get_show_ignore_only(), "show_subed_only": self._get_show_subed_only(), + "version": settings.TA_VERSION, } @@ -329,8 +331,11 @@ class AboutView(View): @staticmethod def get(request): """handle http get""" - colors = AppConfig(request.user.id).colors - context = {"title": "About", "colors": colors} + context = { + "title": "About", + "colors": AppConfig(request.user.id).colors, + "version": settings.TA_VERSION, + } return render(request, "home/about.html", context) @@ -690,6 +695,7 @@ class VideoView(View): "title": video_title, "colors": colors, "cast": cast, + "version": settings.TA_VERSION, } return render(request, "home/video.html", context) @@ -746,7 +752,10 @@ class SearchView(ArchivistResultsView): all_styles = self.get_all_view_styles() self.context.update({"all_styles": all_styles}) self.context.update( - {"search_form": MultiSearchForm(initial=all_styles)} + { + "search_form": MultiSearchForm(initial=all_styles), + "version": settings.TA_VERSION, + } ) return render(request, "home/search.html", self.context) @@ -778,6 +787,7 @@ class SettingsView(View): "user_form": user_form, "app_form": app_form, "scheduler_form": scheduler_form, + "version": settings.TA_VERSION, } return render(request, "home/settings.html", context) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 409ebe5..af75bb3 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,10 +1,10 @@ beautifulsoup4==4.10.0 -celery==5.2.3 +celery==5.2.6 Django==4.0.3 django-cors-headers==3.11.0 djangorestframework==3.13.1 -Pillow==9.0.1 -redis==4.2.1 +Pillow==9.1.0 +redis==4.2.2 requests==2.27.1 ryd-client==0.0.3 uWSGI==2.0.20