From 4853a8b12bbaca309c86bb236998c55be03413eb Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 14 Apr 2022 16:18:08 +0700 Subject: [PATCH 01/18] limit video_daily to below 10k in daily refresh --- tubearchivist/home/src/index/reindex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b254c7f..ed29e89 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -41,6 +41,9 @@ class Reindex: """get daily refresh values""" total_videos = self._get_total_hits("ta_video") video_daily = ceil(total_videos / self.interval * self.MULTIPLY) + if video_daily >= 10000: + video_daily = 9999 + total_channels = self._get_total_hits("ta_channel") channel_daily = ceil(total_channels / self.interval * self.MULTIPLY) total_playlists = self._get_total_hits("ta_playlist") From 80c76c22ca629c1d2999193c8e523aec44ac475b Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 14 Apr 2022 16:49:27 +0700 Subject: [PATCH 02/18] add wiki documentation for sponsorblock integration --- docs/Channels.md | 3 ++- docs/Settings.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Channels.md b/docs/Channels.md index b82c660..a091760 100644 --- a/docs/Channels.md +++ b/docs/Channels.md @@ -28,4 +28,5 @@ Each channel will get a dedicated channel detail page accessible at `/channel/ Date: Fri, 15 Apr 2022 15:03:43 +0700 Subject: [PATCH 03/18] guarantee stats in mapping for sort --- tubearchivist/home/src/es/index_mapping.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 3abf2c1..9c33f72 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -193,6 +193,22 @@ } } }, + "stats" : { + "properties" : { + "average_rating" : { + "type" : "float" + }, + "dislike_count" : { + "type" : "long" + }, + "like_count" : { + "type" : "long" + }, + "view_count" : { + "type" : "long" + } + } + }, "subtitles": { "properties": { "ext": { From d4b1d97f5c5dcb28faf7daac59b4e52dc977f41a Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 15:16:31 +0700 Subject: [PATCH 04/18] add mappings for sponsorblock segments --- tubearchivist/home/src/es/index_mapping.json | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 9c33f72..f023eef 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -245,6 +245,31 @@ }, "is_enabled": { "type": "boolean" + }, + "segments" : { + "properties" : { + "UUID" : { + "type": "keyword" + }, + "actionType" : { + "type": "keyword" + }, + "category" : { + "type": "keyword" + }, + "locked" : { + "type" : "short" + }, + "segment" : { + "type" : "float" + }, + "videoDuration" : { + "type" : "float" + }, + "votes" : { + "type" : "long" + } + } } } } From dc29c6718c92ec4c24cc955a917579aec6664a53 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Fri, 15 Apr 2022 09:21:24 +0100 Subject: [PATCH 05/18] Fix: #210 - Download UI Box being removed when file is still processing (#219) * Fix: https://github.com/bbilly1/tubearchivist/issues/210 - Download UI element is removed when the file has finished downloading, but is still being moved on disk to the storage destination. This means the user is presented with nothing in the UI yet the file is still being processed. * FIX: Resolving linting issue * FIX: Set moving message to never expire and then a second message after the move with a 4 second expiry timer. Co-authored-by: Rob Ainsworth --- tubearchivist/home/src/download/yt_dlp_handler.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 693d0e1..5f09907 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -181,7 +181,22 @@ class VideoDownloader: youtube_id, video_overwrites=self.video_overwrites ) self.channels.add(vid_dict["channel"]["channel_id"]) + mess_dict = { + "status": "message:download", + "level": "info", + "title": "Moving....", + "message": "Moving downloaded file to storage folder", + } + RedisArchivist().set_message("message:download", mess_dict, False) + self.move_to_archive(vid_dict) + mess_dict = { + "status": "message:download", + "level": "info", + "title": "Completed", + "message": "", + } + RedisArchivist().set_message("message:download", mess_dict, 4) self._delete_from_pending(youtube_id) # post processing From c5746089f5fdda6c5d57c43cfd68cec483fdd947 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 17:46:16 +0700 Subject: [PATCH 06/18] auto dedect new es version --- deploy.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy.sh b/deploy.sh index 0818e7a..38f7403 100755 --- a/deploy.sh +++ b/deploy.sh @@ -103,8 +103,9 @@ function validate { # update latest tag compatible es for set and forget function sync_latest_es { - printf "\nsync new es version:\n" - read -r VERSION + VERSION=$(grep "bbilly1/tubearchivist-es" docker-compose.yml | awk '{print $NF}') + printf "\nsync new ES version %s\nContinue?\n" "$VERSION" + read -rn 1 if [[ $(systemctl is-active docker) != 'active' ]]; then echo "starting docker" From 3d16954c79713f6fc34640511588b0324f733a05 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 17:56:17 +0700 Subject: [PATCH 07/18] change to tubearchivist-es, add comments --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a3ac505..89257f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,19 +11,19 @@ services: - media:/youtube - cache:/cache environment: - - ES_URL=http://archivist-es:9200 - - REDIS_HOST=archivist-redis + - ES_URL=http://archivist-es:9200 # needs protocol e.g. http and port + - REDIS_HOST=archivist-redis # don't add protocol - HOST_UID=1000 - HOST_GID=1000 - - TA_USERNAME=tubearchivist - - TA_PASSWORD=verysecret - - ELASTIC_PASSWORD=verysecret - - TZ=America/New_York + - TA_USERNAME=tubearchivist # your initial TA credentials + - TA_PASSWORD=verysecret # your initial TA credentials + - ELASTIC_PASSWORD=verysecret # set password for Elasticsearch + - TZ=America/New_York # set your time zone depends_on: - archivist-es - archivist-redis archivist-redis: - image: redislabs/rejson:latest # For arm64 just update this line with bbilly1/rejson:latest + image: redislabs/rejson:latest # for arm64 use bbilly1/rejson container_name: archivist-redis restart: always expose: @@ -33,12 +33,12 @@ services: depends_on: - archivist-es archivist-es: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 + image: bbilly1/tubearchivist-es # only for amd64, or use official es 7.17.2 container_name: archivist-es restart: always environment: - "xpack.security.enabled=true" - - "ELASTIC_PASSWORD=verysecret" + - "ELASTIC_PASSWORD=verysecret" # matching Elasticsearch password - "discovery.type=single-node" - "ES_JAVA_OPTS=-Xms512m -Xmx512m" ulimits: @@ -46,7 +46,7 @@ services: soft: -1 hard: -1 volumes: - - es:/usr/share/elasticsearch/data + - es:/usr/share/elasticsearch/data # check for permission error when using bind mount, see readme expose: - "9200" From 7231b5d245f9800a44ada70cd4b1e780157cf837 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 20:03:15 +0700 Subject: [PATCH 08/18] implement unset of sb per channel overwrite --- docs/Channels.md | 2 +- tubearchivist/home/src/frontend/forms.py | 1 + tubearchivist/home/templates/home/channel_id.html | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/Channels.md b/docs/Channels.md index a091760..e6f7654 100644 --- a/docs/Channels.md +++ b/docs/Channels.md @@ -29,4 +29,4 @@ Clicking on the *Configure* button will open a form with options to configure se - **Download Format**: Overwrite the download qualities for videos from this channel. - **Auto Delete**: Automatically delete watched videos from this channel after selected days. - **Index Playlists**: Automatically add all Playlists with at least a video downloaded to your index. Only do this for channels where you care about playlists as this will slow down indexing new videos for having to check which playlist this belongs to. -- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can disable or enable SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. \ No newline at end of file +- **SponsorBlock**: Using [SponsorBlock](https://sponsor.ajay.app/) to get and skip sponsored content. Customize per channel: You can *disable* or *enable* SponsorBlock for certain channels only to overwrite the behavior set on the [Settings](settings) page. Selecting *unset* will remove the overwrite and your setting will fall back to the default on the settings page. diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 8709ff1..1a25e6a 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -202,6 +202,7 @@ class ChannelOverwriteForm(forms.Form): ("", "-- change sponsorblock integrations"), ("disable", "disable sponsorblock integration"), ("1", "enable sponsorblock integration"), + ("0", "unset sponsorblock integration"), ] download_format = forms.CharField(label=False, required=False) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index a30b4f5..8240132 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -93,6 +93,8 @@

Enable SponsorBlock: {% if channel_info.channel_overwrites.integrate_sponsorblock %} {{ channel_info.channel_overwrites.integrate_sponsorblock }} + {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %} + Disabled {% else %} False {% endif %}

From c84fbdfb9c251cfc9dff527b6f1e54f9ddc082cb Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 20:04:01 +0700 Subject: [PATCH 09/18] fix _check_get_sb for per channel deactivate --- tubearchivist/home/src/index/video.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index a58a0cb..b8adb9a 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -417,16 +417,15 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def _check_get_sb(self): """check if need to run sponsor block""" - integrate = False - if self.config["downloads"]["integrate_sponsorblock"]: - integrate = True + integrate = self.config["downloads"]["integrate_sponsorblock"] if self.video_overwrites: single_overwrite = self.video_overwrites.get(self.youtube_id) if not single_overwrite: return integrate - integrate = single_overwrite.get("integrate_sponsorblock", False) + if "integrate_sponsorblock" in single_overwrite: + return single_overwrite.get("integrate_sponsorblock") return integrate From 2fc4ed419e4fbc2344db463b12f21d78238cf169 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 20:07:59 +0700 Subject: [PATCH 10/18] add note about tubearchivist-es to readme --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1b80457..b22a505 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,8 @@ Once your YouTube video collection grows, it becomes hard to search and find a specific video. That's where Tube Archivist comes in: By indexing your video collection with metadata from YouTube, you can organize, search and enjoy your archived YouTube videos without hassle offline through a convenient web interface. ## Connect -- [Discord](https://discord.gg/AFwz8nE7BK): Connect with us on our brand new Discord server. -- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our brand new Subreddit. +- [Discord](https://discord.gg/AFwz8nE7BK): Connect with us on our Discord server. +- [r/TubeArchivist](https://www.reddit.com/r/TubeArchivist/): Join our Subreddit. ## Installing and updating Take a look at the example `docker-compose.yml` file provided. Use the *latest* or the named semantic version tag. The *unstable* tag is for intermediate testing and as the name implies, is **unstable** and not be used on your main installation but in a [testing environment](CONTRIBUTING.md). @@ -76,7 +76,9 @@ Should that not be an option, the Tube Archivist container takes these two addit Changing any of these two environment variables will change the files *nginx.conf* and *uwsgi.ini* at startup using `sed` in your container. ### Elasticsearch -**Note**: Newest Tube Archivist depends on Elasticsearch version 7.17 to provide an automatic updatepath. +**Note**: Newest Tube Archivist depends on Elasticsearch version 7.17 to provide an automatic updatepath in the future. + +Use `bbilly1/tubearchivist-es` to automatically get the recommended version, or use the official image with the version tag in the docker-compose file. Stores video meta data and makes everything searchable. Also keeps track of the download queue. - Needs to be accessible over the default port `9200` @@ -98,7 +100,7 @@ For some architectures it might be required to run Redis JSON on a nonstandard p ### Updating Tube Archivist You will see the current version number of **Tube Archivist** in the footer of the interface so you can compare it with the latest release to make sure you are running the *latest and greatest*. * There can be breaking changes between updates, particularly as the application grows, new environment variables or settings might be required for you to set in the your docker-compose file. *Always* check the **release notes**: Any breaking changes will be marked there. -* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. +* All testing and development is done with the Elasticsearch version number as mentioned in the provided *docker-compose.yml* file. This will be updated when a new release of Elasticsearch is available. Running an older version of Elasticsearch is most likely not going to result in any issues, but it's still recommended to run the same version as mentioned. Use `bbilly1/tubearchivist-es` to automatically get the recommended version. ### Alternative installation instructions: - **arm64**: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64. From fe610fdacaf626d1f8d0fab3f9bffb5e150d95ba Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 20:41:38 +0700 Subject: [PATCH 11/18] add log output for backup progress --- tubearchivist/home/src/es/index_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubearchivist/home/src/es/index_setup.py b/tubearchivist/home/src/es/index_setup.py index cd408a0..85c4e28 100644 --- a/tubearchivist/home/src/es/index_setup.py +++ b/tubearchivist/home/src/es/index_setup.py @@ -377,6 +377,7 @@ def backup_all_indexes(reason): for index in backup_handler.index_config: index_name = index["index_name"] + print(f"backup: export in progress for {index_name}") if not backup_handler.index_exists(index_name): continue all_results = backup_handler.get_all_documents(index_name) From f05c2921706098cabf98b977abfa8c682bc4c253 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 21:35:44 +0700 Subject: [PATCH 12/18] add new home message --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b22a505..4da77e7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@

Your self hosted YouTube media server

+Tube Archivist has a new home: https://github.com/tubearchivist/tubearchivist + ## Table of contents: * [Wiki](https://github.com/bbilly1/tubearchivist/wiki) for a detailed documentation, with [FAQ](https://github.com/bbilly1/tubearchivist/wiki/FAQ) * [Core functionality](#core-functionality) From 51ceffd58f7f9cc0cb816b2bb0317cae84b97a93 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 15 Apr 2022 22:35:13 +0700 Subject: [PATCH 13/18] update all links to tubearchivist organization --- CONTRIBUTING.md | 2 +- README.md | 18 +++++++++--------- deploy.sh | 2 +- docs/Installation.md | 4 ++-- tubearchivist/config/settings.py | 2 +- tubearchivist/home/templates/home/about.html | 8 ++++---- tubearchivist/home/templates/home/base.html | 2 +- tubearchivist/home/templates/home/login.html | 2 +- .../home/templates/home/settings.html | 4 ++-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06034d5..f2538fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ If you haven't already, the best place to start is the README. This will give yo ## Report a bug -If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/bbilly1/tubearchivist/issues). +If you notice something is not working as expected, check to see if it has been previously reported in the [open issues](https://github.com/tubearchivist/tubearchivist/issues). If it has not yet been disclosed, go ahead and create an issue. If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. diff --git a/README.md b/README.md index 4da77e7..907efd5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Tube Archivist has a new home: https://github.com/tubearchivist/tubearchivist ## Table of contents: -* [Wiki](https://github.com/bbilly1/tubearchivist/wiki) for a detailed documentation, with [FAQ](https://github.com/bbilly1/tubearchivist/wiki/FAQ) +* [Wiki](https://github.com/tubearchivist/tubearchivist/wiki) for a detailed documentation, with [FAQ](https://github.com/tubearchivist/tubearchivist/wiki/FAQ) * [Core functionality](#core-functionality) * [Screenshots](#screenshots) * [Problem Tube Archivist tries to solve](#problem-tube-archivist-tries-to-solve) @@ -106,7 +106,7 @@ You will see the current version number of **Tube Archivist** in the footer of t ### Alternative installation instructions: - **arm64**: The Tube Archivist container is multi arch, so is Elasticsearch. RedisJSON doesn't offer arm builds, you can use `bbilly1/rejson`, an unofficial rebuild for arm64. -- **Synology**: There is a [discussion thread](https://github.com/bbilly1/tubearchivist/discussions/48) with Synology installation instructions. +- **Synology**: There is a [discussion thread](https://github.com/tubearchivist/tubearchivist/discussions/48) with Synology installation instructions. - **Unraid**: The three containers needed are all in the Community Applications. First install `TubeArchivist RedisJSON` followed by `TubeArchivist ES`, and finally you can install `TubeArchivist`. If you have unraid specific issues, report those to the [support thread](https://forums.unraid.net/topic/114073-support-crocs-tube-archivist/ "support thread"). - **Helm Chart**: There is a Helm Chart available at https://github.com/insuusvenerati/helm-charts. Mostly self-explanatory but feel free to ask questions in the discord / subreddit. @@ -152,20 +152,20 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] User roles - [ ] Podcast mode to serve channel as mp3 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage -- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/bbilly1/tubearchivist/issues/97)) +- [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97)) - [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration -- [ ] Add passing browser cookies to yt-dlp ([#199](https://github.com/bbilly1/tubearchivist/issues/199)) -- [ ] User created playlists, random and repeat controls ([#108](https://github.com/bbilly1/tubearchivist/issues/108), [#220](https://github.com/bbilly1/tubearchivist/issues/220)) +- [ ] Add passing browser cookies to yt-dlp ([#199](https://github.com/tubearchivist/tubearchivist/issues/199)) +- [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220)) - [ ] Auto play or play next link - [ ] Show similar videos on video page - [ ] Multi language support - [ ] Show total video downloaded vs total videos available in channel - [ ] Make items in grid row configurable to use more of the screen - [ ] Add statistics of index -- [ ] Implement complete offline media file import from json file ([#138](https://github.com/bbilly1/tubearchivist/issues/138)) -- [ ] Filter and query in search form, search by url query ([#134](https://github.com/bbilly1/tubearchivist/issues/134), [#139](https://github.com/bbilly1/tubearchivist/issues/139)) -- [ ] Auto ignore videos by keyword ([#163](https://github.com/bbilly1/tubearchivist/issues/163)) -- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/bbilly1/tubearchivist/issues/144)) +- [ ] Implement complete offline media file import from json file ([#138](https://github.com/tubearchivist/tubearchivist/issues/138)) +- [ ] Filter and query in search form, search by url query ([#134](https://github.com/tubearchivist/tubearchivist/issues/134), [#139](https://github.com/tubearchivist/tubearchivist/issues/139)) +- [ ] Auto ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163)) +- [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144)) Implemented: - [X] Implement per channel settings [2022-03-26] diff --git a/deploy.sh b/deploy.sh index 38f7403..92ca8de 100755 --- a/deploy.sh +++ b/deploy.sh @@ -12,7 +12,7 @@ # docker buildx inspect --bootstrap # more details: -# https://github.com/bbilly1/tubearchivist/issues/6 +# https://github.com/tubearchivist/tubearchivist/issues/6 set -e diff --git a/docs/Installation.md b/docs/Installation.md index e15d784..a6cb554 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -13,7 +13,7 @@ Tube Archivist, and all if it's dependencies are located in the [community appli ![enter image description here](https://i.imgur.com/ycAqFRU.png) This is the easiest container to setup of the thee, just make sure that you do not have any port conflicts, and that your `/data` is mounted to the correct path. The other containers will map to the same directory. -If you need to install `TubeArchivist-RedisJSON`on a different port, you'll have to follow [these steps](https://github.com/bbilly1/tubearchivist#redis-on-a-custom-port) later on when installing the `TubeArchivist` container +If you need to install `TubeArchivist-RedisJSON`on a different port, you'll have to follow [these steps](https://github.com/tubearchivist/tubearchivist#redis-on-a-custom-port) later on when installing the `TubeArchivist` container ### Step 2: Install `TubeArchivist-ES` @@ -34,7 +34,7 @@ It's finally time to set up TubeArchivist! Make sure that this is an empty directory to not cause confusion when starting the application. If you have existing videos that you'd like to import into Tube Archivist, please checkout the [settings - wiki.](https://github.com/bbilly1/tubearchivist/wiki/Settings#manual-media-files-import) + wiki.](https://github.com/tubearchivist/tubearchivist/wiki/Settings#manual-media-files-import) - `Appdata:` This should be the same base path as the other two containers. diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 251ad5c..1e92937 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -153,5 +153,5 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ ] # TA application settings -TA_UPSTREAM = "https://github.com/bbilly1/tubearchivist" +TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" TA_VERSION = "v0.1.3" diff --git a/tubearchivist/home/templates/home/about.html b/tubearchivist/home/templates/home/about.html index 26d2416..23e01a8 100644 --- a/tubearchivist/home/templates/home/about.html +++ b/tubearchivist/home/templates/home/about.html @@ -7,13 +7,13 @@

Useful Links

-

This project is in active and constant development, take a look at the roadmap for a overview.

-

For any questions on what a button or a function does, You can find the up-to-date user documentation on Github.

-

All contributions are welcome: Open an issue for any bugs and errors, start a discussion for anything that will require a more indepth look. The contributing page is a good place to get started.

+

This project is in active and constant development, take a look at the roadmap for a overview.

+

For any questions on what a button or a function does, You can find the up-to-date user documentation on Github.

+

All contributions are welcome: Open an issue for any bugs and errors, join us on Discord to discuss details. The contributing page is a good place to get started.

Donate

-

Here are some links, if you want to buy the developer a coffee. Thank you for your support!

+

Here are some links, if you want to buy the developer a coffee. Thank you for your support!

{% endblock content %} diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index ceb35d5..83b41f9 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -132,7 +132,7 @@
- © 2021 - TubeArchivist {{ version }} About | GitHub | Docker Hub | Discord | Reddit + © 2021 - TubeArchivist {{ version }} About | GitHub | Docker Hub | Discord | Reddit
diff --git a/tubearchivist/home/templates/home/login.html b/tubearchivist/home/templates/home/login.html index b19f52a..ecefb42 100644 --- a/tubearchivist/home/templates/home/login.html +++ b/tubearchivist/home/templates/home/login.html @@ -45,7 +45,7 @@ - +
diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 58e69cb..0ff1516 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -258,7 +258,7 @@

Manual media files import.

-

Add files to the cache/import folder. Make sure to follow the instructions in the Github Wiki.

+

Add files to the cache/import folder. Make sure to follow the instructions in the Github Wiki.

@@ -303,7 +303,7 @@

Rescan filesystem

Danger Zone: This will delete the metadata of deleted videos from the filesystem.

-

Rescan your media folder looking for missing videos and clean up index. More infos on the Github Wiki.

+

Rescan your media folder looking for missing videos and clean up index. More infos on the Github Wiki.

From 1cf82bdbbdf973f45a3bee5b654e22a68c6731bb Mon Sep 17 00:00:00 2001 From: Nathan DeTar Date: Fri, 15 Apr 2022 18:53:15 -0700 Subject: [PATCH 14/18] Cleanup & Prevent repeat of skip message. (#227) * Cleanup & Prevent repeat of skip message. * skip sponsorblock segments if not indexed * disable onVideoProgress sb check if not indexed Co-authored-by: simon --- tubearchivist/static/script.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 4031ec0..4287100 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -333,7 +333,7 @@ function createPlayer(button) { var videoData = getVideoData(videoId); var sponsorBlockElements = ''; - if (videoData.data.sponsorblock.is_enabled) { + if (videoData.data.sponsorblock && videoData.data.sponsorblock.is_enabled) { sponsorBlock = videoData.data.sponsorblock; if (sponsorBlock.segments.length == 0) { sponsorBlockElements = ` @@ -505,19 +505,21 @@ function getVideoPlayerWatchStatus() { return watched; } -// Runs on video playback, marks video as watched if video gets to 90% or higher, sends position to api +// Runs on video playback, marks video as watched if video gets to 90% or higher, sends position to api, SB skipping function onVideoProgress() { var videoId = getVideoPlayerVideoId(); var currentTime = getVideoPlayerCurrentTime(); var duration = getVideoPlayerDuration(); var videoElement = getVideoPlayer(); - // var sponsorBlockElement = document.getElementById("sponsorblock"); var notificationsElement = document.getElementById("notifications"); - if (sponsorBlock.segments.length > 0) { + if (sponsorBlock && sponsorBlock.segments) { for(let i in sponsorBlock.segments) { - if(sponsorBlock.segments[i].segment[0] <= currentTime + 0.3 && sponsorBlock.segments[i].segment[0] >= currentTime) { + if(currentTime >= sponsorBlock.segments[i].segment[0] && currentTime <= sponsorBlock.segments[i].segment[0] + 0.3) { videoElement.currentTime = sponsorBlock.segments[i].segment[1]; - notificationsElement.innerHTML += `

Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.

`; + var notificationElement = document.getElementById("notification-" + sponsorBlock.segments[i].UUID); + if (!notificationElement) { + notificationsElement.innerHTML += `

Skipped sponsor segment from ${formatTime(sponsorBlock.segments[i].segment[0])} to ${formatTime(sponsorBlock.segments[i].segment[1])}.

`; + } } if(currentTime > sponsorBlock.segments[i].segment[1] + 10) { var notificationsElementUUID = document.getElementById("notification-" + sponsorBlock.segments[i].UUID); From 59f1c111aa0382d03df0bc665e1b65c34e502e6d Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 16 Apr 2022 08:58:35 +0700 Subject: [PATCH 15/18] extend timeout for sleep interval --- tubearchivist/home/src/download/yt_dlp_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 5f09907..31d8ce9 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -196,7 +196,7 @@ class VideoDownloader: "title": "Completed", "message": "", } - RedisArchivist().set_message("message:download", mess_dict, 4) + RedisArchivist().set_message("message:download", mess_dict, 10) self._delete_from_pending(youtube_id) # post processing From 70ac33928f48dc0cfe18ba22b424a7245165c7a1 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 16 Apr 2022 09:29:49 +0700 Subject: [PATCH 16/18] update roadmap --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 907efd5..d3ee806 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,9 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] Podcast mode to serve channel as mp3 - [ ] Implement [PyFilesystem](https://github.com/PyFilesystem/pyfilesystem2) for flexible video storage - [ ] Implement [Apprise](https://github.com/caronc/apprise) for notifications ([#97](https://github.com/tubearchivist/tubearchivist/issues/97)) -- [ ] Add [SponsorBlock](https://sponsor.ajay.app/) integration - [ ] Add passing browser cookies to yt-dlp ([#199](https://github.com/tubearchivist/tubearchivist/issues/199)) - [ ] User created playlists, random and repeat controls ([#108](https://github.com/tubearchivist/tubearchivist/issues/108), [#220](https://github.com/tubearchivist/tubearchivist/issues/220)) -- [ ] Auto play or play next link +- [ ] Auto play or play next link ([#226](https://github.com/tubearchivist/tubearchivist/issues/226)) - [ ] Show similar videos on video page - [ ] Multi language support - [ ] Show total video downloaded vs total videos available in channel @@ -166,8 +165,10 @@ We have come far, nonetheless we are not short of ideas on how to improve and ex - [ ] Filter and query in search form, search by url query ([#134](https://github.com/tubearchivist/tubearchivist/issues/134), [#139](https://github.com/tubearchivist/tubearchivist/issues/139)) - [ ] Auto ignore videos by keyword ([#163](https://github.com/tubearchivist/tubearchivist/issues/163)) - [ ] Custom searchable notes to videos, channels, playlists ([#144](https://github.com/tubearchivist/tubearchivist/issues/144)) +- [ ] Download video comments Implemented: +- [X] Add [SponsorBlock](https://sponsor.ajay.app/) integration [2022-04-16] - [X] Implement per channel settings [2022-03-26] - [X] Subtitle download & indexing [2022-02-13] - [X] Fancy advanced unified search interface [2022-01-08] From fcdb31e848087d0e18dbc976825fe3e9615d5f33 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 16 Apr 2022 09:30:40 +0700 Subject: [PATCH 17/18] bump release number --- tubearchivist/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 1e92937..8e8bf5a 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -154,4 +154,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.1.3" +TA_VERSION = "v0.1.4" From 50f4e0e5b70fb45d69cd0637ac28eca463a84132 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 16 Apr 2022 13:08:32 +0700 Subject: [PATCH 18/18] skip base64 generator, hotfix --- tubearchivist/home/src/index/video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index b8adb9a..969d716 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -10,7 +10,6 @@ from datetime import datetime import requests from django.conf import settings -from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap from home.src.index import channel as ta_channel from home.src.index.generic import YouTubeItem @@ -437,7 +436,8 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): upload_date_time = datetime.strptime(upload_date, "%Y%m%d") published = upload_date_time.strftime("%Y-%m-%d") last_refresh = int(datetime.now().strftime("%s")) - base64_blur = ThumbManager().get_base64_blur(self.youtube_id) + # base64_blur = ThumbManager().get_base64_blur(self.youtube_id) + base64_blur = False # build json_data basics self.json_data = { "title": self.youtube_meta["title"],