From bbb0c26498296c7eeb33b8d98fd945d14d31601e Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 29 May 2022 11:43:25 +0700 Subject: [PATCH 1/8] fix subtitle delete overwrite, change filename to .lang.vtt, #195 --- tubearchivist/home/src/index/reindex.py | 3 ++- tubearchivist/home/src/index/video.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 5fbdeee..5e85f95 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -153,6 +153,7 @@ class Reindex: date_downloaded = video.json_data["date_downloaded"] channel_dict = video.json_data["channel"] playlist = video.json_data.get("playlist") + subtitles = video.json_data.get("subtitles") # get new video.build_json() @@ -160,7 +161,7 @@ class Reindex: video.deactivate() return - video.delete_subtitles() + video.delete_subtitles(subtitles=subtitles) video.check_subtitles() # add back diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index fa2b0f9..86d44a0 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -67,7 +67,7 @@ class YoutubeSubtitle: return False video_media_url = self.video.json_data["media_url"] - media_url = video_media_url.replace(".mp4", f"-{lang}.vtt") + media_url = video_media_url.replace(".mp4", f".{lang}.vtt") all_formats = all_subtitles.get(lang) if not all_formats: return False @@ -103,7 +103,7 @@ class YoutubeSubtitle: return False video_media_url = self.video.json_data["media_url"] - media_url = video_media_url.replace(".mp4", f"-{lang}.vtt") + media_url = video_media_url.replace(".mp4", f".{lang}.vtt") all_formats = all_subtitles.get(lang) if not all_formats: # no user subtitles found @@ -152,15 +152,19 @@ class YoutubeSubtitle: """send subtitle to es for indexing""" _, _ = ElasticWrap("_bulk").post(data=query_str, ndjson=True) - def delete(self): + def delete(self, subtitles=False): """delete subtitles from index and filesystem""" youtube_id = self.video.youtube_id - # delete files videos_base = self.video.config["application"]["videos"] - if not self.video.json_data.get("subtitles"): - return + # delete files + if subtitles: + files = [i["media_url"] for i in subtitles] + else: + if not self.video.json_data.get("subtitles"): + return + + files = [i["media_url"] for i in self.video.json_data["subtitles"]] - files = [i["media_url"] for i in self.video.json_data["subtitles"]] for file_name in files: file_path = os.path.join(videos_base, file_name) try: @@ -594,10 +598,10 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): ) playlist.upload_to_es() - def delete_subtitles(self): + def delete_subtitles(self, subtitles=False): """delete indexed subtitles""" print(f"{self.youtube_id}: delete subtitles") - YoutubeSubtitle(self).delete() + YoutubeSubtitle(self).delete(subtitles=subtitles) def _get_ryd_stats(self): """get optional stats from returnyoutubedislikeapi.com""" From 25e2bdb0b780dd1c7bfef6e439d76b6c9d88bba8 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 14:51:44 +0700 Subject: [PATCH 2/8] bump celery version --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 322992f..a1e49c9 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4==4.11.1 -celery==5.2.6 +celery==5.2.7 Django==4.0.4 django-cors-headers==3.12.0 djangorestframework==3.13.1 From a9b8d34b5a6be3ebb9802f0f4037b18cc9864780 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 14:52:47 +0700 Subject: [PATCH 3/8] adapt search width with grid_items --- tubearchivist/home/templates/home/search.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/templates/home/search.html b/tubearchivist/home/templates/home/search.html index 8372c61..a4b2620 100644 --- a/tubearchivist/home/templates/home/search.html +++ b/tubearchivist/home/templates/home/search.html @@ -1,7 +1,7 @@ {% extends "home/base.html" %} {% block content %}
-
+

Search your Archive

@@ -10,7 +10,7 @@

Video Results

-
+

No videos found.

From c6d696a91b5a2b44e6abbd496a35e92aec2261a5 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 15:29:52 +0700 Subject: [PATCH 4/8] timeout search as you type --- tubearchivist/static/script.js | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 9c016be..96ad261 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -785,21 +785,25 @@ function setProgressBar(videoId, currentTime, duration) { } // multi search form +let searchTimeout = null; function searchMulti(query) { - if (query.length > 1) { - var payload = JSON.stringify({'multi_search': query}); - var http = new XMLHttpRequest(); - http.onreadystatechange = function() { - if (http.readyState === 4) { - allResults = JSON.parse(http.response).results; - populateMultiSearchResults(allResults); - } - }; - http.open("POST", "/process/", true); - http.setRequestHeader("X-CSRFToken", getCookie("csrftoken")); - http.setRequestHeader("Content-type", "application/json"); - http.send(payload); - } + clearTimeout(searchTimeout); + searchTimeout = setTimeout(function () { + if (query.length > 1) { + var payload = JSON.stringify({'multi_search': query}); + var http = new XMLHttpRequest(); + http.onreadystatechange = function() { + if (http.readyState === 4) { + allResults = JSON.parse(http.response).results; + populateMultiSearchResults(allResults); + } + }; + http.open("POST", "/process/", true); + http.setRequestHeader("X-CSRFToken", getCookie("csrftoken")); + http.setRequestHeader("Content-type", "application/json"); + http.send(payload); + } + }, 500); } function getViewDefaults(view) { From c93e935b9c6f9091df9540ad30076bacd66e1578 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 16:06:12 +0700 Subject: [PATCH 5/8] move delete download queue button to settings page --- tubearchivist/home/templates/home/downloads.html | 9 --------- tubearchivist/home/templates/home/settings.html | 6 ++++++ tubearchivist/static/script.js | 8 ++++---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 23dd98a..3ca469e 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -45,15 +45,6 @@ list view
-
- {% if show_ignored_only %} -

Ignored from download

- - {% else %} -

Download queue

- - {% endif %} -

Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}

{% if results %} diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index d4cd7f5..a5bb82a 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -269,6 +269,12 @@

Actions

+
+

Delete download queue

+

Delete your pending or previously ignored videos from your download queue.

+ + +

Manual media files import.

Add files to the cache/import folder. Make sure to follow the instructions in the Github Wiki.

diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 96ad261..47ef727 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -203,10 +203,10 @@ function deleteQueue(button) { var to_delete = button.getAttribute('data-id'); var payload = JSON.stringify({'deleteQueue': to_delete}); sendPost(payload); - setTimeout(function(){ - location.reload(); - return false; - }, 1000); + // clear button + var message = document.createElement('p'); + message.innerText = 'deleting download queue: ' + to_delete; + document.getElementById(button.id).replaceWith(message); } function stopQueue() { From cbb714c23534b19d8db8674e2288e95de32a4493 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 16:21:24 +0700 Subject: [PATCH 6/8] show grid count controls only if applicable --- tubearchivist/home/templates/home/channel_id.html | 9 ++++++--- tubearchivist/home/templates/home/home.html | 9 ++++++--- tubearchivist/home/templates/home/playlist_id.html | 9 ++++++--- tubearchivist/static/css/style.css | 10 ++++++++-- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 04a4333..5aefcba 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -145,9 +145,12 @@ sort-icon {% if view_style == "grid" %}
- {{ grid_items }} - grid plus row - grid minus row + {% if grid_items < 7 %} + grid plus row + {% endif %} + {% if grid_items > 3 %} + grid minus row + {% endif %}
{% endif %} grid view diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index fc4c76a..cc49756 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -76,9 +76,12 @@ sort-icon {% if view_style == "grid" %}
- {{ grid_items }} - grid plus row - grid minus row + {% if grid_items < 7 %} + grid plus row + {% endif %} + {% if grid_items > 3 %} + grid minus row + {% endif %}
{% endif %} grid view diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 60b5522..1f5569c 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -80,9 +80,12 @@
{% if view_style == "grid" %}
- {{ grid_items }} - grid plus row - grid minus row + {% if grid_items < 7 %} + grid plus row + {% endif %} + {% if grid_items > 3 %} + grid minus row + {% endif %}
{% endif %} grid view diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 5ed455d..f97e2f2 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -346,14 +346,20 @@ button:hover { justify-content: end; } -.view-icons img, -.grid-count img { +.view-icons img { width: 30px; margin: 5px 10px; cursor: pointer; filter: var(--img-filter); } +.grid-count img { + width: 15px; + margin: 5px; + cursor: pointer; + filter: var(--img-filter); +} + #hidden-form { display: none; } From 231ac1e012af3961f8c498af023d3cd5ba4b9606 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 17:59:52 +0700 Subject: [PATCH 7/8] make channel matching optional for reusability --- tubearchivist/home/src/download/queue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 47225dd..c115fbc 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -269,9 +269,12 @@ class PendingList(PendingIndex): "vid_thumb_url": vid["thumbnail"], "title": vid["title"], "channel_id": vid["channel_id"], - "channel_indexed": vid["channel_id"] in self.all_channels, "duration": duration_str, "published": published, "timestamp": int(datetime.now().strftime("%s")), } + if self.all_channels: + youtube_details.update( + {"channel_indexed": vid["channel_id"] in self.all_channels} + ) return youtube_details From 5d262d7d485b80984c7f91da9b330bb0e6652fc5 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 30 May 2022 18:22:17 +0700 Subject: [PATCH 8/8] skip cookie if no config added to YtWrap --- tubearchivist/home/src/download/yt_dlp_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index b32d536..260b955 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -30,7 +30,8 @@ class YtWrap: """build yt-dlp obs""" self.obs = self.OBS_BASE.copy() self.obs.update(self.obs_request) - self.add_cookie() + if self.config: + self.add_cookie() def add_cookie(self): """add cookie if enabled"""