Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}
+
Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}
Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_ryd }}
-
Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}
+
Integrate with SponsorBlock to get sponsored timestamps: {{ config.downloads.integrate_sponsorblock }}
Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_sponsorblock }}
From 9d73dbc45a2dfe990695003a7a52cd7f52f6d028 Mon Sep 17 00:00:00 2001
From: simon
Date: Fri, 8 Apr 2022 17:19:25 +0700
Subject: [PATCH 09/16] implement video overwrites index for sponsorblock
---
.../home/src/download/yt_dlp_handler.py | 4 +++-
tubearchivist/home/src/index/video.py | 22 +++++++++++++++----
2 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py
index d764011..693d0e1 100644
--- a/tubearchivist/home/src/download/yt_dlp_handler.py
+++ b/tubearchivist/home/src/download/yt_dlp_handler.py
@@ -177,7 +177,9 @@ class VideoDownloader:
except yt_dlp.utils.DownloadError:
print("failed to download " + youtube_id)
continue
- vid_dict = index_new_video(youtube_id)
+ vid_dict = index_new_video(
+ youtube_id, video_overwrites=self.video_overwrites
+ )
self.channels.add(vid_dict["channel"]["channel_id"])
self.move_to_archive(vid_dict)
self._delete_from_pending(youtube_id)
diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py
index a728caf..28ad536 100644
--- a/tubearchivist/home/src/index/video.py
+++ b/tubearchivist/home/src/index/video.py
@@ -358,9 +358,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
index_name = "ta_video"
yt_base = "https://www.youtube.com/watch?v="
- def __init__(self, youtube_id):
+ def __init__(self, youtube_id, video_overwrites=False):
super().__init__(youtube_id)
self.channel_id = False
+ self.video_overwrites = video_overwrites
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
def build_json(self):
@@ -377,11 +378,24 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
if self.config["downloads"]["integrate_ryd"]:
self._get_ryd_stats()
- if self.config["downloads"]["integrate_sponsorblock"]:
+ if self._check_get_sb():
self._get_sponsorblock()
return
+ def _check_get_sb(self):
+ """check if need to run sponsor block"""
+ if self.config["downloads"]["integrate_sponsorblock"]:
+ return True
+ try:
+ single_overwrite = self.video_overwrites[self.youtube_id]
+ _ = single_overwrite["integrate_sponsorblock"]
+ return True
+ except KeyError:
+ return False
+
+ return False
+
def _process_youtube_meta(self):
"""extract relevant fields from youtube"""
# extract
@@ -547,9 +561,9 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
_, _ = ElasticWrap(path).post(data=data)
-def index_new_video(youtube_id):
+def index_new_video(youtube_id, video_overwrites=False):
"""combined classes to create new video in index"""
- video = YoutubeVideo(youtube_id)
+ video = YoutubeVideo(youtube_id, video_overwrites=video_overwrites)
video.build_json()
if not video.json_data:
raise ValueError("failed to get metadata for " + youtube_id)
From 552636d8825a88734c8bf19d1f56071f22828254 Mon Sep 17 00:00:00 2001
From: simon
Date: Fri, 8 Apr 2022 17:55:24 +0700
Subject: [PATCH 10/16] handle 404 in video item API view
---
tubearchivist/api/views.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py
index d938b2f..4ad216e 100644
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@@ -86,7 +86,8 @@ class VideoApiView(ApiBaseView):
# pylint: disable=unused-argument
"""get request"""
self.get_document(video_id)
- self.process_keys()
+ if self.response.get("data"):
+ self.process_keys()
return Response(self.response, status=self.status_code)
From 236215cc4c7ac6f0b328ede8981f370bda01e642 Mon Sep 17 00:00:00 2001
From: simon
Date: Fri, 8 Apr 2022 22:10:16 +0700
Subject: [PATCH 11/16] add config var to video.html template
---
tubearchivist/home/views.py | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py
index 0cd2a58..92359b8 100644
--- a/tubearchivist/home/views.py
+++ b/tubearchivist/home/views.py
@@ -670,7 +670,7 @@ class VideoView(View):
def get(self, request, video_id):
"""get single video"""
- colors, cast = self.read_config(user_id=request.user.id)
+ config_handler = AppConfig(request.user.id)
path = f"ta_video/_doc/{video_id}"
look_up = SearchHandler(path, config=False)
video_hit = look_up.get_data()
@@ -692,9 +692,10 @@ class VideoView(View):
"video": video_data,
"playlist_nav": playlist_nav,
"title": video_title,
- "colors": colors,
- "cast": cast,
+ "colors": config_handler.colors,
+ "cast": config_handler.config["application"]["enable_cast"],
"version": settings.TA_VERSION,
+ "config": config_handler.config,
}
return render(request, "home/video.html", context)
@@ -711,14 +712,6 @@ class VideoView(View):
return all_navs
- @staticmethod
- def read_config(user_id):
- """read config file"""
- config_handler = AppConfig(user_id)
- cast = config_handler.config["application"]["enable_cast"]
- colors = config_handler.colors
- return colors, cast
-
@staticmethod
def star_creator(rating):
"""convert rating float to stars"""
From 99e0c1c90ef89542457f8521cf4bad96e71d0bed Mon Sep 17 00:00:00 2001
From: simon
Date: Fri, 8 Apr 2022 23:15:58 +0700
Subject: [PATCH 12/16] bump yt-dlp
---
tubearchivist/requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt
index 1650ebd..d7e9c47 100644
--- a/tubearchivist/requirements.txt
+++ b/tubearchivist/requirements.txt
@@ -9,4 +9,4 @@ requests==2.27.1
ryd-client==0.0.3
uWSGI==2.0.20
whitenoise==6.0.0
-yt_dlp==2022.3.8.2
+yt_dlp==2022.4.8
From d8c7b3df0bf259f5458369ba64320b79926247a4 Mon Sep 17 00:00:00 2001
From: simon
Date: Sat, 9 Apr 2022 14:12:03 +0700
Subject: [PATCH 13/16] better error message for wrong es version, #197
---
tubearchivist/home/apps.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/tubearchivist/home/apps.py b/tubearchivist/home/apps.py
index 46a940c..1053bc6 100644
--- a/tubearchivist/home/apps.py
+++ b/tubearchivist/home/apps.py
@@ -97,9 +97,8 @@ class StartupCheck:
if invalid:
print(
- "minial required elasticsearch version: "
- + f"{self.MIN_MAJOR}.{self.MIN_MINOR}, "
- + "please update to recommended version."
+ "required elasticsearch version: "
+ + f"{self.MIN_MAJOR}.{self.MIN_MINOR}"
)
sys.exit(1)
From 10385b14142bb4e147052fddd5c244842a2c9220 Mon Sep 17 00:00:00 2001
From: simon
Date: Sun, 10 Apr 2022 15:57:15 +0700
Subject: [PATCH 14/16] bump bs4
---
tubearchivist/requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt
index d7e9c47..7d03d08 100644
--- a/tubearchivist/requirements.txt
+++ b/tubearchivist/requirements.txt
@@ -1,4 +1,4 @@
-beautifulsoup4==4.11.0
+beautifulsoup4==4.11.1
celery==5.2.6
Django==4.0.3
django-cors-headers==3.11.0
From 31378ac756695e6a5f7acb9426487f61a8561289 Mon Sep 17 00:00:00 2001
From: simon
Date: Sun, 10 Apr 2022 15:58:11 +0700
Subject: [PATCH 15/16] better generic schedule validator for impossible input,
#209
---
tubearchivist/home/src/ta/config.py | 33 +++++++++++++++++++++--------
1 file changed, 24 insertions(+), 9 deletions(-)
diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py
index c89d62c..4b98c4a 100644
--- a/tubearchivist/home/src/ta/config.py
+++ b/tubearchivist/home/src/ta/config.py
@@ -220,18 +220,33 @@ class ScheduleBuilder:
raise ValueError("invalid input")
to_write = dict(zip(keys, values))
- all_hours = [int(i) for i in re.split(r"\D+", to_write["hour"])]
- if max(all_hours) > 23:
- print("hour can't be greater than 23")
- raise ValueError("invalid input")
- try:
- int(to_write["minute"])
- except ValueError as error:
- print("too frequent: only number in minutes are supported")
- raise ValueError("invalid input") from error
+ self._validate_cron(to_write)
return to_write
+ @staticmethod
+ def _validate_cron(to_write):
+ """validate all fields, raise value error for impossible schedule"""
+ all_hours = list(re.split(r"\D+", to_write["hour"]))
+ for hour in all_hours:
+ if hour.isdigit() and int(hour) > 23:
+ print("hour can not be greater than 23")
+ raise ValueError("invalid input")
+
+ all_days = list(re.split(r"\D+", to_write["day_of_week"]))
+ for day in all_days:
+ if day.isdigit() and int(day) > 6:
+ print("day can not be greater than 6")
+ raise ValueError("invalid input")
+
+ if not to_write["minute"].isdigit():
+ print("too frequent: only number in minutes are supported")
+ raise ValueError("invalid input")
+
+ if int(to_write["minute"]) > 59:
+ print("minutes can not be greater than 59")
+ raise ValueError("invalid input")
+
def build_schedule(self):
"""build schedule dict as expected by app.conf.beat_schedule"""
schedule_dict = {}
From c316d0554957fc4900e2ec94c02c82ecf979fae9 Mon Sep 17 00:00:00 2001
From: Nathan DeTar
Date: Sun, 10 Apr 2022 02:20:58 -0700
Subject: [PATCH 16/16] Added sponsorblock skipping (#208)
* Added sponsorblock skipping.
* Basic framework for sending SB timestamps
* Sponsorblock send timestamp UI improvements
* Added Sponsorblock Icons
* Minor UI tweaks
* Revert UI changes, implement in new UI
* Added notification when sponsor segment is skipped
* Add formatting for notifications & SB messages
* Added SB messages to JS player
* Added SB skip notifcation to videos page.
* Added SB messages to video page
* Change SB messages.
* Check channel_overwrites
* Check Per Channel Settings.
* Cleanup
---
tubearchivist/home/templates/home/video.html | 31 ++++
tubearchivist/static/css/style.css | 19 +++
tubearchivist/static/script.js | 150 ++++++++++++++++++-
3 files changed, 199 insertions(+), 1 deletion(-)
diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html
index 324d293..d1ff3a7 100644
--- a/tubearchivist/home/templates/home/video.html
+++ b/tubearchivist/home/templates/home/video.html
@@ -3,6 +3,36 @@
{% load static %}
{% load humanize %}
+
+
+ {% if video.channel.channel_overwrites.integrate_sponsorblock %}
+ {% if video.channel.channel_overwrites.integrate_sponsorblock == True %}
+ {% if not video.sponsorblock %}
+
This video doesn't have any sponsor segments added. To add a segment go to this video on YouTube and add a segment using the SponsorBlock extension.
+ {% endif %}
+ {% if video.sponsorblock %}
+ {% for segment in video.sponsorblock %}
+ {% if segment.locked != 1 %}
+