From 9a31243b1cba82dd9b811776f807560a49dfc288 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 11:11:09 +0700 Subject: [PATCH 1/7] log message when skipping video --- tubearchivist/home/src/download/queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 36e3d1e..47225dd 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -172,6 +172,8 @@ class PendingList(PendingIndex): """add video to list""" if url not in self.missing_videos and url not in self.to_skip: self.missing_videos.append(url) + else: + print(f"{url}: skipped adding already indexed video to download.") def _parse_channel(self, url): """add all videos of channel to list""" From 5982932457c1afa645f07b68939530e560ee8672 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 13:57:29 +0700 Subject: [PATCH 2/7] implement variable grid row size --- tubearchivist/home/config.json | 3 +- tubearchivist/home/src/frontend/api_calls.py | 12 +++ .../home/templates/home/channel_id.html | 11 ++- tubearchivist/home/templates/home/home.html | 13 +++- tubearchivist/home/views.py | 10 +++ tubearchivist/static/css/style.css | 25 ++++++- tubearchivist/static/img/icon-substract.svg | 75 +++++++++++++++++++ tubearchivist/static/script.js | 10 +++ 8 files changed, 151 insertions(+), 8 deletions(-) create mode 100644 tubearchivist/static/img/icon-substract.svg diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index 9ffe376..3e5e941 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -8,7 +8,8 @@ "home": "grid", "channel": "list", "downloads": "list", - "playlist": "grid" + "playlist": "grid", + "grid_items": 3 }, "subscriptions": { "auto_search": false, diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index 8b0d9b3..ed937ae 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -54,6 +54,7 @@ class PostData: "watched": self._watched, "un_watched": self._un_watched, "change_view": self._change_view, + "change_grid": self._change_grid, "rescan_pending": self._rescan_pending, "ignore": self._ignore, "dl_pending": self._dl_pending, @@ -100,6 +101,17 @@ class PostData: RedisArchivist().set_message(key, {"status": new_view}, expire=False) return {"success": True} + def _change_grid(self): + """process change items in grid""" + grid_items = int(self.exec_val) + grid_items = max(grid_items, 3) + grid_items = min(grid_items, 7) + + key = f"{self.current_user}:grid_items" + print(f"change grid items: {grid_items}") + RedisArchivist().set_message(key, {"status": grid_items}, expire=False) + return {"success": True} + @staticmethod def _rescan_pending(): """look for new items in subscribed channels""" diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 8240132..d0c5bd5 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -112,6 +112,8 @@ {% endif %} + +
Hide watched videos: @@ -141,14 +143,19 @@
sort-icon + {% if view_style == "grid" %} + {{ grid_items }} + grid plus row + grid minus row + {% endif %} grid view list view
-
-
+
+
{% if results %} {% for video in results %}
diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index 445de5b..89dd702 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -1,12 +1,12 @@ {% extends "home/base.html" %} {% block content %} {% load static %} -
+
{% if continue_vids %}

Continue Watching

-
+
{% for video in continue_vids %}
@@ -74,14 +74,19 @@
sort-icon + {% if view_style == "grid" %} + {{ grid_items }} + grid plus row + grid minus row + {% endif %} grid view list view
-
-
+
+
{% if results %} {% for video in results %}
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index bd5a2ea..30738d2 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -77,6 +77,15 @@ class ArchivistViewConfig(View): return view_style + def _get_grid_items(self): + """return items per row to show in grid view""" + grid_key = f"{self.user_id}:grid_items" + grid_items = self.user_conf.get_message(grid_key)["status"] + if not grid_items: + grid_items = self.default_conf["default_view"]["grid_items"] + + return grid_items + def get_all_view_styles(self): """get dict of all view stiles for search form""" all_keys = ["channel", "playlist", "home"] @@ -120,6 +129,7 @@ class ArchivistViewConfig(View): "sort_by": self._get_sort_by(), "sort_order": self._get_sort_order(), "view_style": self._get_view_style(), + "grid_items": self._get_grid_items(), "hide_watched": self._get_hide_watched(), "show_ignored_only": self._get_show_ignore_only(), "show_subed_only": self._get_show_subed_only(), diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index d92187a..808c2c2 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -133,6 +133,13 @@ button:hover { margin: 0 auto; } +.boxed-content.boxed-5, +.boxed-content.boxed-6, +.boxed-content.boxed-7 { + max-width: unset; + width: 85%; +} + .round-img img { border-radius: 50%; } @@ -395,10 +402,26 @@ button:hover { margin-top: 1rem; } -.video-list.grid { +.video-list.grid.grid-3 { grid-template-columns: 1fr 1fr 1fr; } +.video-list.grid.grid-4 { + grid-template-columns: 1fr 1fr 1fr 1fr; +} + +.video-list.grid.grid-5 { + grid-template-columns: 1fr 1fr 1fr 1fr 1fr; +} + +.video-list.grid.grid-6 { + grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr; +} + +.video-list.grid.grid-7 { + grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr 1fr; +} + .video-list.list { grid-template-columns: unset; } diff --git a/tubearchivist/static/img/icon-substract.svg b/tubearchivist/static/img/icon-substract.svg new file mode 100644 index 0000000..0b68cfb --- /dev/null +++ b/tubearchivist/static/img/icon-substract.svg @@ -0,0 +1,75 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index e20d31c..9c016be 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -121,6 +121,16 @@ function changeView(image) { }, 500); } +function changeGridItems(image) { + var operator = image.getAttribute("data-value"); + var payload = JSON.stringify({'change_grid': operator}); + sendPost(payload); + setTimeout(function(){ + location.reload(); + return false; + }, 500); +} + function toggleCheckbox(checkbox) { // pass checkbox id as key and checkbox.checked as value var toggleId = checkbox.id; From caaed252f9ec60eb6cf91db5728c6e52d82ce8e3 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 14:34:01 +0700 Subject: [PATCH 3/7] specify videos page for get_last_youtube_videos of channel --- tubearchivist/home/src/download/subscriptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 973c76e..916c170 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -43,7 +43,8 @@ class ChannelSubscription: if limit: obs["playlistend"] = self.config["subscriptions"]["channel_size"] - channel = YtWrap(obs, self.config).extract(channel_id) + url = f"https://www.youtube.com/channel/{channel_id}/videos" + channel = YtWrap(obs, self.config).extract(url) if not channel: return False From e638f8b487872fdc74ac5501aaa6a5692bf179c7 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 17:47:21 +0700 Subject: [PATCH 4/7] skip subtitle segment without segs key, #249 --- tubearchivist/home/src/index/video.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 7b951d8..acf097f 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -189,9 +189,9 @@ class SubtitleParser: self.all_cues = [] for idx, event in enumerate(all_events): - if "dDurationMs" not in event: - # some events won't have a duration - print(f"failed to parse event without duration: {event}") + if "dDurationMs" not in event or "segs" not in event: + # some events won't have a duration or segs + print(f"skipping subtitle event without content: {event}") continue cue = { @@ -215,15 +215,16 @@ class SubtitleParser: if flatten: # fix overlapping retiming issue - if "dDurationMs" not in flatten[-1]: - # some events won't have a duration - print(f"failed to parse event without duration: {event}") + last = flatten[-1] + if "dDurationMs" not in last or "segs" not in last: + # some events won't have a duration or segs + print(f"skipping subtitle event without content: {event}") continue - last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"] + last_end = last["tStartMs"] + last["dDurationMs"] if event["tStartMs"] < last_end: - joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text - flatten[-1]["segs"][0]["utf8"] = joined + joined = last["segs"][0]["utf8"] + "\n" + text + last["segs"][0]["utf8"] = joined continue event.update({"segs": [{"utf8": text}]}) From 9f92bf95b4226beff3cdb6ebba4381c150692b9e Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 21:23:56 +0700 Subject: [PATCH 5/7] mark video as downloaded false when deleting video --- tubearchivist/home/src/index/video.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index acf097f..fa2b0f9 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -12,6 +12,7 @@ import requests from django.conf import settings from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel +from home.src.index import playlist as ta_playlist from home.src.index.generic import YouTubeItem from home.src.ta.helper import ( DurationConverter, @@ -561,6 +562,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def delete_media_file(self): """delete video file, meta data""" + print(f"{self.youtube_id}: delete video") self.get_from_es() video_base = self.app_conf["videos"] media_url = self.json_data.get("media_url") @@ -570,9 +572,28 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): except FileNotFoundError: print(f"{self.youtube_id}: failed {media_url}, continue.") + self.del_in_playlists() self.del_in_es() self.delete_subtitles() + def del_in_playlists(self): + """remove downloaded in playlist""" + all_playlists = self.json_data.get("playlist") + if not all_playlists: + return + + for playlist_id in all_playlists: + print(f"{playlist_id}: delete video {self.youtube_id}") + playlist = ta_playlist.YoutubePlaylist(playlist_id) + playlist.get_from_es() + entries = playlist.json_data["playlist_entries"] + for idx, entry in enumerate(entries): + if entry["youtube_id"] == self.youtube_id: + playlist.json_data["playlist_entries"][idx].update( + {"downloaded": False} + ) + playlist.upload_to_es() + def delete_subtitles(self): """delete indexed subtitles""" print(f"{self.youtube_id}: delete subtitles") From 0759b4f22011e7ef1f6049f13d13491834243ce8 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 21:41:17 +0700 Subject: [PATCH 6/7] reset mobile variable gird row count --- .../home/templates/home/channel_id.html | 2 ++ tubearchivist/home/templates/home/home.html | 8 +++--- tubearchivist/static/css/style.css | 26 ++++++++++++++++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index d0c5bd5..04a4333 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -144,9 +144,11 @@
sort-icon {% if view_style == "grid" %} +
{{ grid_items }} grid plus row grid minus row +
{% endif %} grid view list view diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index 89dd702..fc4c76a 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -75,9 +75,11 @@
sort-icon {% if view_style == "grid" %} - {{ grid_items }} - grid plus row - grid minus row +
+ {{ grid_items }} + grid plus row + grid minus row +
{% endif %} grid view list view diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 808c2c2..5ed455d 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -133,6 +133,11 @@ button:hover { margin: 0 auto; } +.boxed-content.boxed-4 { + max-width: 1200px; + width: 80%; +} + .boxed-content.boxed-5, .boxed-content.boxed-6, .boxed-content.boxed-7 { @@ -335,12 +340,14 @@ button:hover { margin: 15px 0; } -.view-icons { +.view-icons, +.grid-count { display: flex; justify-content: end; } -.view-icons img { +.view-icons img, +.grid-count img { width: 30px; margin: 5px 10px; cursor: pointer; @@ -1077,7 +1084,11 @@ button:hover { .boxed-content { width: 90%; } - .video-list.grid, + .video-list.grid.grid-3, + .video-list.grid.grid-4, + .video-list.grid.grid-5, + .video-list.grid.grid-6, + .video-list.grid.grid-7, .dl-list.grid, .channel-list.grid, .playlist-list.grid { @@ -1107,6 +1118,9 @@ button:hover { position: unset; transform: unset; } + .grid-count { + display: none; + } .video-player { height: unset; padding: 20px 0 @@ -1121,7 +1135,11 @@ button:hover { * { word-wrap: anywhere; } - .video-list.grid, + .video-list.grid.grid-3, + .video-list.grid.grid-4, + .video-list.grid.grid-5, + .video-list.grid.grid-6, + .video-list.grid.grid-7, .dl-list.grid, .channel-list.grid, .video-item.list, From 51a45bb59ed60831c77e3f163001b61c76c63651 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 28 May 2022 21:48:15 +0700 Subject: [PATCH 7/7] implement variable grid row size for playlist_id --- tubearchivist/home/templates/home/playlist_id.html | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index f9e987e..60b5522 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -63,6 +63,8 @@
{% endif %} +
+
Hide watched videos: @@ -76,14 +78,21 @@
+ {% if view_style == "grid" %} +
+ {{ grid_items }} + grid plus row + grid minus row +
+ {% endif %} grid view list view
-
-
+
+
{% if results %} {% for video in results %}