From b1a7a6a148f07c68daeb18015971306c4acdab69 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 7 Apr 2022 22:30:20 +0700 Subject: [PATCH 01/16] use cleaned_data for config form parser --- tubearchivist/home/src/ta/config.py | 62 ++++++++++++++--------------- tubearchivist/home/views.py | 35 ++++++++-------- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 3258ed0..724e612 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -83,33 +83,32 @@ class AppConfig: def update_config(self, form_post): """update config values from settings form""" - config = self.config for key, value in form_post.items(): - to_write = value[0] - if len(to_write): - if to_write == "0": - to_write = False - elif to_write == "1": - to_write = True - elif to_write.isdigit(): - to_write = int(to_write) + if not value and not isinstance(value, int): + continue - config_dict, config_value = key.split("_", maxsplit=1) - config[config_dict][config_value] = to_write + if value in ["0", 0]: + to_write = False + elif value == "1": + to_write = True + else: + to_write = value - RedisArchivist().set_message("config", config, expire=False) + config_dict, config_value = key.split("_", maxsplit=1) + self.config[config_dict][config_value] = to_write + + RedisArchivist().set_message("config", self.config, expire=False) @staticmethod def set_user_config(form_post, user_id): """set values in redis for user settings""" for key, value in form_post.items(): - to_write = value[0] - if len(to_write): - if to_write.isdigit(): - to_write = int(to_write) - message = {"status": to_write} - redis_key = f"{user_id}:{key}" - RedisArchivist().set_message(redis_key, message, expire=False) + if not value: + continue + + message = {"status": value} + redis_key = f"{user_id}:{key}" + RedisArchivist().set_message(redis_key, message, expire=False) def get_colors(self): """overwrite config if user has set custom values""" @@ -172,12 +171,11 @@ class ScheduleBuilder: print("processing form, restart container for changes to take effect") redis_config = self.config for key, value in form_post.items(): - to_check = value[0] - if key in self.SCHEDULES and to_check: + if key in self.SCHEDULES and value: try: - to_write = self.value_builder(key, to_check) + to_write = self.value_builder(key, value) except ValueError: - print(f"failed: {key} {to_check}") + print(f"failed: {key} {value}") mess_dict = { "status": "message:setting", "level": "error", @@ -188,8 +186,8 @@ class ScheduleBuilder: return redis_config["scheduler"][key] = to_write - if key in self.CONFIG and to_check: - redis_config["scheduler"][key] = int(to_check) + if key in self.CONFIG and value: + redis_config["scheduler"][key] = int(value) RedisArchivist().set_message("config", redis_config, expire=False) mess_dict = { "status": "message:setting", @@ -199,26 +197,26 @@ class ScheduleBuilder: } RedisArchivist().set_message("message:setting", mess_dict) - def value_builder(self, key, to_check): + def value_builder(self, key, value): """validate single cron form entry and return cron dict""" - print(f"change schedule for {key} to {to_check}") - if to_check == "0": + print(f"change schedule for {key} to {value}") + if value == "0": # deactivate this schedule return False - if re.search(r"[\d]{1,2}\/[\d]{1,2}", to_check): + if re.search(r"[\d]{1,2}\/[\d]{1,2}", value): # number/number cron format will fail in celery print("number/number schedule formatting not supported") raise ValueError keys = ["minute", "hour", "day_of_week"] - if to_check == "auto": + if value == "auto": # set to sensible default values = self.SCHEDULES[key].split() else: - values = to_check.split() + values = value.split() if len(keys) != len(values): - print(f"failed to parse {to_check} for {key}") + print(f"failed to parse {value} for {key}") raise ValueError("invalid input") to_write = dict(zip(keys, values)) diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 19d1cfe..e7bd143 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -8,7 +8,6 @@ import json import urllib.parse from time import sleep -from django import forms from django.conf import settings from django.contrib.auth import login from django.contrib.auth.forms import AuthenticationForm @@ -802,23 +801,25 @@ class SettingsView(View): @staticmethod def post(request): """handle form post to update settings""" + user_form = UserSettingsForm(request.POST) + if user_form.is_valid(): + user_form_post = user_form.cleaned_data + if any(user_form_post.values()): + AppConfig().set_user_config(user_form_post, request.user.id) - form_response = forms.Form(request.POST) - if form_response.is_valid(): - form_post = dict(request.POST) - print(form_post) - del form_post["csrfmiddlewaretoken"] - config_handler = AppConfig() - if "application-settings" in form_post: - del form_post["application-settings"] - config_handler.update_config(form_post) - elif "user-settings" in form_post: - del form_post["user-settings"] - config_handler.set_user_config(form_post, request.user.id) - elif "scheduler-settings" in form_post: - del form_post["scheduler-settings"] - print(form_post) - ScheduleBuilder().update_schedule_conf(form_post) + app_form = ApplicationSettingsForm(request.POST) + if app_form.is_valid(): + app_form_post = app_form.cleaned_data + if app_form_post: + print(app_form_post) + AppConfig().update_config(app_form_post) + + scheduler_form = SchedulerSettingsForm(request.POST) + if scheduler_form.is_valid(): + scheduler_form_post = scheduler_form.cleaned_data + if any(scheduler_form_post.values()): + print(scheduler_form_post) + ScheduleBuilder().update_schedule_conf(scheduler_form_post) sleep(1) return redirect("settings", permanent=True) From 6a1cb15114af581e3e215627ee9d894106a1c048 Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 7 Apr 2022 23:02:07 +0700 Subject: [PATCH 02/16] validate hours to not be greater than 23, #209 --- tubearchivist/home/src/ta/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 724e612..c89d62c 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -220,6 +220,10 @@ class ScheduleBuilder: raise ValueError("invalid input") to_write = dict(zip(keys, values)) + all_hours = [int(i) for i in re.split(r"\D+", to_write["hour"])] + if max(all_hours) > 23: + print("hour can't be greater than 23") + raise ValueError("invalid input") try: int(to_write["minute"]) except ValueError as error: From a07d789e66bacff04586e9c030c7216d57745a23 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 00:29:09 +0700 Subject: [PATCH 03/16] add base64 blur video thumb --- tubearchivist/home/src/download/thumbnails.py | 19 ++++++++++++++++++- tubearchivist/home/src/es/index_mapping.json | 4 ++++ tubearchivist/home/src/index/video.py | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index 0ce492f..d25f4d1 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -4,8 +4,10 @@ functionality: - check for missing thumbnails """ +import base64 import os from collections import Counter +from io import BytesIO from time import sleep import requests @@ -15,7 +17,7 @@ from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist from home.src.ta.ta_redis import RedisArchivist from mutagen.mp4 import MP4, MP4Cover -from PIL import Image +from PIL import Image, ImageFilter class ThumbManager: @@ -241,6 +243,21 @@ class ThumbManager: } RedisArchivist().set_message("message:download", mess_dict) + def get_base64_blur(self, youtube_id): + """return base64 encoded placeholder""" + img_path = self.vid_thumb_path(youtube_id) + file_path = os.path.join(self.CACHE_DIR, img_path) + img_raw = Image.open(file_path) + img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20)) + img_blur = img_raw.filter(ImageFilter.BLUR) + buffer = BytesIO() + img_blur.save(buffer, format="JPEG") + img_data = buffer.getvalue() + img_base64 = base64.b64encode(img_data).decode() + data_url = f"data:image/jpg;base64,{img_base64}" + + return data_url + @staticmethod def vid_thumb_path(youtube_id): """build expected path for video thumbnail from youtube_id""" diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 29f6b7e..0fddc11 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -73,6 +73,10 @@ "type": "text", "index": false }, + "vid_thumb_base64": { + "type": "text", + "index": false + }, "date_downloaded": { "type": "date" }, diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 411b4af..811059e 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -10,6 +10,7 @@ from datetime import datetime import requests from django.conf import settings +from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem @@ -389,12 +390,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): upload_date_time = datetime.strptime(upload_date, "%Y%m%d") published = upload_date_time.strftime("%Y-%m-%d") last_refresh = int(datetime.now().strftime("%s")) + base64_blur = ThumbManager().get_base64_blur(self.youtube_id) # build json_data basics self.json_data = { "title": self.youtube_meta["title"], "description": self.youtube_meta["description"], "category": self.youtube_meta["categories"], "vid_thumb_url": self.youtube_meta["thumbnail"], + "vid_thumb_base64": base64_blur, "tags": self.youtube_meta["tags"], "published": published, "vid_last_refresh": last_refresh, From e2f4dd124a89d3ab2bb3450a8c37a04e1bbf8256 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 15:15:37 +0700 Subject: [PATCH 04/16] bump beautifulsoup4 --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index af75bb3..1650ebd 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4==4.10.0 +beautifulsoup4==4.11.0 celery==5.2.6 Django==4.0.3 django-cors-headers==3.11.0 From efa240440b14f925dce428b62638bee2c487d166 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 15:15:59 +0700 Subject: [PATCH 05/16] fix userspace for vide base config --- tubearchivist/home/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index e7bd143..0cd2a58 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -111,7 +111,7 @@ class ArchivistViewConfig(View): """build default context for every view""" self.user_id = user_id self.user_conf = RedisArchivist() - self.default_conf = AppConfig().config + self.default_conf = AppConfig(self.user_id).config self.context = { "colors": self.default_conf["application"]["colors"], From fe3e3cfaccb80aa8d168761cc75da44faafba671 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 15:56:34 +0700 Subject: [PATCH 06/16] fix filenotfounderror for missing subtitles when deleting video --- tubearchivist/home/src/index/video.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 811059e..a728caf 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -498,7 +498,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): for media_url in to_del: file_path = os.path.join(video_base, media_url) - os.remove(file_path) + try: + os.remove(file_path) + except FileNotFoundError: + print(f"{self.youtube_id}: failed {media_url}, continue.") self.del_in_es() self.delete_subtitles() From 365ebf53a5709c200118d74e753cc00e1a81783b Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 17:17:39 +0700 Subject: [PATCH 07/16] implement per channel sponsorblock --- tubearchivist/home/src/es/index_mapping.json | 6 ++++++ tubearchivist/home/src/frontend/forms.py | 9 +++++++++ tubearchivist/home/src/index/channel.py | 7 ++++++- tubearchivist/home/templates/home/channel_id.html | 10 +++++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 0fddc11..9a0ee47 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -50,6 +50,9 @@ }, "index_playlists": { "type": "boolean" + }, + "integrate_sponsorblock": { + "type" : "boolean" } } } @@ -130,6 +133,9 @@ }, "index_playlists": { "type": "boolean" + }, + "integrate_sponsorblock": { + "type" : "boolean" } } } diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 53ef284..337a805 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -198,8 +198,17 @@ class ChannelOverwriteForm(forms.Form): ("1", "Enable playlist index"), ] + SP_CHOICES = [ + ("", "-- change sponsorblock integrations"), + ("0", "disable sponsorblock integration"), + ("1", "enable sponsorblock integration"), + ] + download_format = forms.CharField(label=False, required=False) autodelete_days = forms.IntegerField(label=False, required=False) index_playlists = forms.ChoiceField( widget=forms.Select, choices=PLAYLIST_INDEX, required=False ) + integrate_sponsorblock = forms.ChoiceField( + widget=forms.Select, choices=SP_CHOICES, required=False + ) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 953078d..75824d8 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -340,7 +340,12 @@ class YoutubeChannel(YouTubeItem): def set_overwrites(self, overwrites): """set per channel overwrites""" - valid_keys = ["download_format", "autodelete_days", "index_playlists"] + valid_keys = [ + "download_format", + "autodelete_days", + "index_playlists", + "integrate_sponsorblock", + ] to_write = self.json_data.get("channel_overwrites", {}) for key, value in overwrites.items(): diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 80b090b..a30b4f5 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -89,7 +89,15 @@ {% endif %}

{{ channel_overwrite_form.index_playlists }}
-
+
+

Enable SponsorBlock: + {% if channel_info.channel_overwrites.integrate_sponsorblock %} + {{ channel_info.channel_overwrites.integrate_sponsorblock }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.integrate_sponsorblock }}
+
From 859bf2a28dc89f9729dcdfed67348ef956f94452 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 17:18:33 +0700 Subject: [PATCH 08/16] add target _blank to external links --- tubearchivist/home/templates/home/settings.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index e4d9534..4fba379 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -124,12 +124,12 @@
-

Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}

+

Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}

Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_ryd }}
-

Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}

+

Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}

Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_sponsorblock }}
From 9d73dbc45a2dfe990695003a7a52cd7f52f6d028 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 17:19:25 +0700 Subject: [PATCH 09/16] implement video overwrites index for sponsorblock --- .../home/src/download/yt_dlp_handler.py | 4 +++- tubearchivist/home/src/index/video.py | 22 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index d764011..693d0e1 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -177,7 +177,9 @@ class VideoDownloader: except yt_dlp.utils.DownloadError: print("failed to download " + youtube_id) continue - vid_dict = index_new_video(youtube_id) + vid_dict = index_new_video( + youtube_id, video_overwrites=self.video_overwrites + ) self.channels.add(vid_dict["channel"]["channel_id"]) self.move_to_archive(vid_dict) self._delete_from_pending(youtube_id) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index a728caf..28ad536 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -358,9 +358,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): index_name = "ta_video" yt_base = "https://www.youtube.com/watch?v=" - def __init__(self, youtube_id): + def __init__(self, youtube_id, video_overwrites=False): super().__init__(youtube_id) self.channel_id = False + self.video_overwrites = video_overwrites self.es_path = f"{self.index_name}/_doc/{youtube_id}" def build_json(self): @@ -377,11 +378,24 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if self.config["downloads"]["integrate_ryd"]: self._get_ryd_stats() - if self.config["downloads"]["integrate_sponsorblock"]: + if self._check_get_sb(): self._get_sponsorblock() return + def _check_get_sb(self): + """check if need to run sponsor block""" + if self.config["downloads"]["integrate_sponsorblock"]: + return True + try: + single_overwrite = self.video_overwrites[self.youtube_id] + _ = single_overwrite["integrate_sponsorblock"] + return True + except KeyError: + return False + + return False + def _process_youtube_meta(self): """extract relevant fields from youtube""" # extract @@ -547,9 +561,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): _, _ = ElasticWrap(path).post(data=data) -def index_new_video(youtube_id): +def index_new_video(youtube_id, video_overwrites=False): """combined classes to create new video in index""" - video = YoutubeVideo(youtube_id) + video = YoutubeVideo(youtube_id, video_overwrites=video_overwrites) video.build_json() if not video.json_data: raise ValueError("failed to get metadata for " + youtube_id) From 552636d8825a88734c8bf19d1f56071f22828254 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 17:55:24 +0700 Subject: [PATCH 10/16] handle 404 in video item API view --- tubearchivist/api/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index d938b2f..4ad216e 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -86,7 +86,8 @@ class VideoApiView(ApiBaseView): # pylint: disable=unused-argument """get request""" self.get_document(video_id) - self.process_keys() + if self.response.get("data"): + self.process_keys() return Response(self.response, status=self.status_code) From 236215cc4c7ac6f0b328ede8981f370bda01e642 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 22:10:16 +0700 Subject: [PATCH 11/16] add config var to video.html template --- tubearchivist/home/views.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 0cd2a58..92359b8 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -670,7 +670,7 @@ class VideoView(View): def get(self, request, video_id): """get single video""" - colors, cast = self.read_config(user_id=request.user.id) + config_handler = AppConfig(request.user.id) path = f"ta_video/_doc/{video_id}" look_up = SearchHandler(path, config=False) video_hit = look_up.get_data() @@ -692,9 +692,10 @@ class VideoView(View): "video": video_data, "playlist_nav": playlist_nav, "title": video_title, - "colors": colors, - "cast": cast, + "colors": config_handler.colors, + "cast": config_handler.config["application"]["enable_cast"], "version": settings.TA_VERSION, + "config": config_handler.config, } return render(request, "home/video.html", context) @@ -711,14 +712,6 @@ class VideoView(View): return all_navs - @staticmethod - def read_config(user_id): - """read config file""" - config_handler = AppConfig(user_id) - cast = config_handler.config["application"]["enable_cast"] - colors = config_handler.colors - return colors, cast - @staticmethod def star_creator(rating): """convert rating float to stars""" From 99e0c1c90ef89542457f8521cf4bad96e71d0bed Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 8 Apr 2022 23:15:58 +0700 Subject: [PATCH 12/16] bump yt-dlp --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 1650ebd..d7e9c47 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -9,4 +9,4 @@ requests==2.27.1 ryd-client==0.0.3 uWSGI==2.0.20 whitenoise==6.0.0 -yt_dlp==2022.3.8.2 +yt_dlp==2022.4.8 From d8c7b3df0bf259f5458369ba64320b79926247a4 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 9 Apr 2022 14:12:03 +0700 Subject: [PATCH 13/16] better error message for wrong es version, #197 --- tubearchivist/home/apps.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py index 46a940c..1053bc6 100644 --- a/tubearchivist/home/apps.py +++ b/tubearchivist/home/apps.py @@ -97,9 +97,8 @@ class StartupCheck: if invalid: print( - "minial required elasticsearch version: " - + f"{self.MIN_MAJOR}.{self.MIN_MINOR}, " - + "please update to recommended version." + "required elasticsearch version: " + + f"{self.MIN_MAJOR}.{self.MIN_MINOR}" ) sys.exit(1) From 10385b14142bb4e147052fddd5c244842a2c9220 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 10 Apr 2022 15:57:15 +0700 Subject: [PATCH 14/16] bump bs4 --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index d7e9c47..7d03d08 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4==4.11.0 +beautifulsoup4==4.11.1 celery==5.2.6 Django==4.0.3 django-cors-headers==3.11.0 From 31378ac756695e6a5f7acb9426487f61a8561289 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 10 Apr 2022 15:58:11 +0700 Subject: [PATCH 15/16] better generic schedule validator for impossible input, #209 --- tubearchivist/home/src/ta/config.py | 33 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index c89d62c..4b98c4a 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -220,18 +220,33 @@ class ScheduleBuilder: raise ValueError("invalid input") to_write = dict(zip(keys, values)) - all_hours = [int(i) for i in re.split(r"\D+", to_write["hour"])] - if max(all_hours) > 23: - print("hour can't be greater than 23") - raise ValueError("invalid input") - try: - int(to_write["minute"]) - except ValueError as error: - print("too frequent: only number in minutes are supported") - raise ValueError("invalid input") from error + self._validate_cron(to_write) return to_write + @staticmethod + def _validate_cron(to_write): + """validate all fields, raise value error for impossible schedule""" + all_hours = list(re.split(r"\D+", to_write["hour"])) + for hour in all_hours: + if hour.isdigit() and int(hour) > 23: + print("hour can not be greater than 23") + raise ValueError("invalid input") + + all_days = list(re.split(r"\D+", to_write["day_of_week"])) + for day in all_days: + if day.isdigit() and int(day) > 6: + print("day can not be greater than 6") + raise ValueError("invalid input") + + if not to_write["minute"].isdigit(): + print("too frequent: only number in minutes are supported") + raise ValueError("invalid input") + + if int(to_write["minute"]) > 59: + print("minutes can not be greater than 59") + raise ValueError("invalid input") + def build_schedule(self): """build schedule dict as expected by app.conf.beat_schedule""" schedule_dict = {} From c316d0554957fc4900e2ec94c02c82ecf979fae9 Mon Sep 17 00:00:00 2001 From: Nathan DeTar Date: Sun, 10 Apr 2022 02:20:58 -0700 Subject: [PATCH 16/16] Added sponsorblock skipping (#208) * Added sponsorblock skipping. * Basic framework for sending SB timestamps * Sponsorblock send timestamp UI improvements * Added Sponsorblock Icons * Minor UI tweaks * Revert UI changes, implement in new UI * Added notification when sponsor segment is skipped * Add formatting for notifications & SB messages * Added SB messages to JS player * Added SB skip notifcation to videos page. * Added SB messages to video page * Change SB messages. * Check channel_overwrites * Check Per Channel Settings. * Cleanup --- tubearchivist/home/templates/home/video.html | 31 ++++ tubearchivist/static/css/style.css | 19 +++ tubearchivist/static/script.js | 150 ++++++++++++++++++- 3 files changed, 199 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index 324d293..d1ff3a7 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -3,6 +3,36 @@ {% load static %} {% load humanize %}
+
+
+ {% if video.channel.channel_overwrites.integrate_sponsorblock %} + {% if video.channel.channel_overwrites.integrate_sponsorblock == True %} + {% if not video.sponsorblock %} +

This video doesn't have any sponsor segments added. To add a segment go to this video on YouTube and add a segment using the SponsorBlock extension.

+ {% endif %} + {% if video.sponsorblock %} + {% for segment in video.sponsorblock %} + {% if segment.locked != 1 %} +

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

+ {{ break }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} + {% elif config.downloads.integrate_sponsorblock %} + {% if not video.sponsorblock %} +

This video doesn't have any sponsor segments added. To add a segment go to this video on YouTube and add a segment using the SponsorBlock extension.

+ {% endif %} + {% if video.sponsorblock %} + {% for segment in video.sponsorblock %} + {% if segment.locked != 1 %} +

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

+ {{ break }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +
{% if cast %} @@ -114,6 +144,7 @@
diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 0ddf32c..d92187a 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -62,6 +62,13 @@ h3 { color: var(--accent-font-light); } +h4 { + font-size: 0.7em; + margin-bottom: 7px; + font-family: Sen-Regular, sans-serif; + color: var(--accent-font-light); +} + p, i, li { font-family: Sen-Regular, sans-serif; margin-bottom: 10px; @@ -355,6 +362,18 @@ button:hover { height: 100vh; } +.notifications { + text-align: center; + width: 80%; + margin: auto; +} + +.sponsorblock { + text-align: center; + width: 80%; + margin: auto; +} + .video-player video, .video-main video { max-height: 80vh; diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index bc9696d..ee95b6f 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -327,9 +327,33 @@ function cancelDelete() { } // player +var sponsorBlock = []; function createPlayer(button) { var videoId = button.getAttribute('data-id'); var videoData = getVideoData(videoId); + + var sponsorBlockElements = ''; + if (videoData.config.downloads.integrate_sponsorblock && (typeof(videoData.data.channel.channel_overwrites) == "undefined" || typeof(videoData.data.channel.channel_overwrites.integrate_sponsorblock) == "undefined" || videoData.data.channel.channel_overwrites.integrate_sponsorblock == true)) { + sponsorBlock = videoData.data.sponsorblock; + if (!sponsorBlock) { + sponsorBlockElements = ` +
+

This video doesn't have any sponsor segments added. To add a segment go to this video on Youtube and add a segment using the SponsorBlock extension.

+
+ `; + } else { + for(let i in sponsorBlock) { + if(sponsorBlock[i].locked != 1) { + sponsorBlockElements = ` +
+

This video has unlocked sponsor segments. Go to this video on YouTube and vote on the segments using the SponsorBlock extension.

+
+ `; + break; + } + } + } + } var videoProgress = getVideoProgress(videoId).position; var videoName = videoData.data.title; @@ -353,7 +377,6 @@ function createPlayer(button) { var channelName = videoData.data.channel.channel_name; removePlayer(); - // document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info // If cast integration is enabled create cast button var castButton = ''; @@ -383,6 +406,8 @@ function createPlayer(button) { const markup = `
${videoTag} +
+ ${sponsorBlockElements}
close-icon ${watchStatusIndicator} @@ -400,6 +425,53 @@ function createPlayer(button) { divPlayer.innerHTML = markup; } +// function sendSponsorBlockVote(uuid, vote) { +// var videoId = getVideoPlayerVideoId(); +// postSponsorSegmentVote(videoId, uuid, vote); +// } + +// var sponsorBlockTimestamps = []; +// function sendSponsorBlockSegment() { +// var videoId = getVideoPlayerVideoId(); +// var currentTime = getVideoPlayerCurrentTime(); +// var sponsorBlockElement = document.getElementById("sponsorblock"); +// if (sponsorBlockTimestamps[1]) { +// if (sponsorBlockTimestamps[1] > sponsorBlockTimestamps[0]) { +// postSponsorSegment(videoId, sponsorBlockTimestamps[0], sponsorBlockTimestamps[1]); +// sponsorBlockElement.innerHTML = ` +//

Timestamps sent! (Not really)

+// `; +// setTimeout(function(){ +// sponsorBlockElement.innerHTML = ` +// +// `; +// }, 3000); +// } else { +// sponsorBlockElement.innerHTML = ` +// Invalid Timestamps! +// `; +// setTimeout(function(){ +// sponsorBlockElement.innerHTML = ` +// +// `; +// }, 3000); +// } +// sponsorBlockTimestamps = []; +// } else if (sponsorBlockTimestamps[0]) { +// sponsorBlockTimestamps.push(currentTime); +// sponsorBlockElement.innerHTML = ` +//

${sponsorBlockTimestamps[0].toFixed(1)} s |

+//

${sponsorBlockTimestamps[1].toFixed(1)} s |

+// +// `; +// } else { +// sponsorBlockTimestamps.push(currentTime); +// sponsorBlockElement.innerHTML = ` +// +// `; +// } +// } + // Add video tag to video page when passed a video id, function loaded on page load `video.html (115-117)` function insertVideoTag(videoData, videoProgress) { var videoTag = createVideoTag(videoData, videoProgress); @@ -488,6 +560,32 @@ function onVideoProgress() { var videoId = getVideoPlayerVideoId(); var currentTime = getVideoPlayerCurrentTime(); var duration = getVideoPlayerDuration(); + var videoElement = getVideoPlayer(); + // var sponsorBlockElement = document.getElementById("sponsorblock"); + var notificationsElement = document.getElementById("notifications"); + if (sponsorBlock) { + for(let i in sponsorBlock) { + if(sponsorBlock[i].segment[0] <= currentTime + 0.3 && sponsorBlock[i].segment[0] >= currentTime) { + videoElement.currentTime = sponsorBlock[i].segment[1]; + notificationsElement.innerHTML += `

Skipped sponsor segment from ${formatTime(sponsorBlock[i].segment[0])} to ${formatTime(sponsorBlock[i].segment[1])}.

`; + } + // if(currentTime >= sponsorBlock[i].segment[1] && currentTime <= sponsorBlock[i].segment[1] + 0.2) { + // if(sponsorBlock[i].locked != 1) { + // sponsorBlockElement.innerHTML += ` + //
+ // + // + //
`; + // } + // } + if(currentTime > sponsorBlock[i].segment[1] + 10) { + var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock[i].UUID); + if(notificationsElementUUID) { + notificationsElementUUID.outerHTML = ''; + } + } + } + } if ((currentTime % 10).toFixed(1) <= 0.2) { // Check progress every 10 seconds or else progress is checked a few times a second postVideoProgress(videoId, currentTime); if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched @@ -542,6 +640,32 @@ function formatNumbers(number) { return numberFormatted; } +// Formats times in seconds for frontend +function formatTime(time) { + var hoursUnformatted = time / 3600; + var minutesUnformatted = (time % 3600) / 60; + var secondsUnformatted = time % 60; + + var hoursFormatted = Math.trunc(hoursUnformatted); + if(minutesUnformatted < 10 && hoursFormatted > 0) { + var minutesFormatted = "0" + Math.trunc(minutesUnformatted); + } else { + var minutesFormatted = Math.trunc(minutesUnformatted); + } + if(secondsUnformatted < 10) { + var secondsFormatted = "0" + Math.trunc(secondsUnformatted); + } else { + var secondsFormatted = Math.trunc(secondsUnformatted); + } + + var timeUnformatted = ''; + if(hoursFormatted > 0) { + timeUnformatted = hoursFormatted + ":" + } + var timeFormatted = timeUnformatted.concat(minutesFormatted, ":", secondsFormatted); + return timeFormatted; +} + // Gets video data when passed video ID function getVideoData(videoId) { var apiEndpoint = "/api/video/" + videoId + "/"; @@ -599,6 +723,30 @@ function postVideoProgress(videoId, videoProgress) { } } +// Send sponsor segment when given video id and and timestamps +function postSponsorSegment(videoId, startTime, endTime) { + var apiEndpoint = "/api/video/" + videoId + "/sponsor/"; + var data = { + "segment": { + "startTime": startTime, + "endTime": endTime + } + }; + apiRequest(apiEndpoint, "POST", data); +} + +// Send sponsor segment when given video id and and timestamps +function postSponsorSegmentVote(videoId, uuid, vote) { + var apiEndpoint = "/api/video/" + videoId + "/sponsor/"; + var data = { + "vote": { + "uuid": uuid, + "yourVote": vote + } + }; + apiRequest(apiEndpoint, "POST", data); +} + // Makes api requests when passed an endpoint and method ("GET", "POST", "DELETE") function apiRequest(apiEndpoint, method, data) { const xhttp = new XMLHttpRequest();