From 1b0be8497224c8a74077e24cbf263cf4b78f604b Mon Sep 17 00:00:00 2001 From: Omar Laham Date: Fri, 13 Oct 2023 03:20:42 +0200 Subject: [PATCH 01/13] Remove /media/ prefix from Download File URL in video.html (#567) --- tubearchivist/home/templates/home/video.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index 698d2a8d..d1e43ae1 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -85,7 +85,7 @@ {% endif %} - +
Are you sure? From e1fce06f9723cfc187fb2f4e398a5b1d9e99f7e9 Mon Sep 17 00:00:00 2001 From: Steve Ovens Date: Sun, 15 Oct 2023 01:58:06 -0500 Subject: [PATCH 02/13] View only user (#539) * Remove repo docs in favor of hosted docs (#537) * updated base, channel, video htmls to hide elements based on if user is staff or in the group 'admin' * added the load auth_extras * updated auth_extras * updated views.py to block api calls from deleting files from unprivileged users; The Templates needed to be updated to support the various group checks related to removing buttons an unprivileged user should not see * bumped the channel templates to remove conflict * fix linting issues * more linting --------- Co-authored-by: Merlin <4706504+MerlinScheurer@users.noreply.github.com> --- tubearchivist/api/views.py | 21 ++++++++++++++++++- tubearchivist/home/templates/home/base.html | 3 +++ .../home/templates/home/channel.html | 3 +++ .../home/templates/home/channel_id.html | 4 ++++ .../templates/home/channel_id_playlist.html | 2 +- .../home/templates/home/playlist.html | 6 ++++++ .../home/templates/home/playlist_id.html | 4 ++++ tubearchivist/home/templates/home/video.html | 5 +++++ tubearchivist/home/templatetags/__init__.py | 0 .../home/templatetags/auth_extras.py | 8 +++++++ 10 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tubearchivist/home/templatetags/__init__.py create mode 100644 tubearchivist/home/templatetags/auth_extras.py diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index a01a8749..9d5eafa1 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -2,6 +2,8 @@ from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress from api.src.search_processor import SearchProcess +from django.contrib.auth.decorators import user_passes_test +from django.utils.decorators import method_decorator from home.src.download.queue import PendingInteract from home.src.download.subscriptions import ( ChannelSubscription, @@ -39,6 +41,11 @@ from rest_framework.response import Response from rest_framework.views import APIView +def check_admin(user): + """if user has access to restricted views""" + return user.is_staff or user.groups.filter(name="admin").exists() + + class ApiBaseView(APIView): """base view to inherit from""" @@ -109,6 +116,7 @@ class VideoApiView(ApiBaseView): self.get_document(video_id) return Response(self.response, status=self.status_code) + @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, video_id): # pylint: disable=unused-argument """delete single video""" @@ -165,7 +173,6 @@ class VideoProgressView(ApiBaseView): message = {"position": position, "youtube_id": video_id} RedisArchivist().set_message(key, message) self.response = request.data - return Response(self.response) def delete(self, request, video_id): @@ -283,6 +290,7 @@ class ChannelApiView(ApiBaseView): self.get_document(channel_id) return Response(self.response, status=self.status_code) + @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, channel_id): # pylint: disable=unused-argument """delete channel""" @@ -328,6 +336,7 @@ class ChannelApiListView(ApiBaseView): return Response(self.response) + @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request): """subscribe/unsubscribe to list of channels""" data = request.data @@ -520,6 +529,7 @@ class DownloadApiView(ApiBaseView): self.get_document(video_id) return Response(self.response, status=self.status_code) + @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request, video_id): """post to video to change status""" item_status = request.data.get("status") @@ -540,6 +550,7 @@ class DownloadApiView(ApiBaseView): return Response(request.data) + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def delete(request, video_id): # pylint: disable=unused-argument @@ -585,6 +596,7 @@ class DownloadApiListView(ApiBaseView): self.get_document_list(request) return Response(self.response) + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """add list of videos to download queue""" @@ -610,6 +622,7 @@ class DownloadApiListView(ApiBaseView): return Response(data) + @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request): """delete download queue""" query_filter = request.GET.get("filter", False) @@ -661,6 +674,7 @@ class LoginApiView(ObtainAuthToken): return Response({"token": token.key, "user_id": user.pk}) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SnapshotApiListView(ApiBaseView): """resolves to /api/snapshot/ GET: returns snapshot config plus list of existing snapshots @@ -684,6 +698,7 @@ class SnapshotApiListView(ApiBaseView): return Response(response) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SnapshotApiView(ApiBaseView): """resolves to /api/snapshot// GET: return a single snapshot @@ -738,6 +753,7 @@ class TaskListView(ApiBaseView): return Response(all_results) +@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskNameListView(ApiBaseView): """resolves to /api/task-name// GET: return a list of stored results of task @@ -776,6 +792,7 @@ class TaskNameListView(ApiBaseView): return Response({"message": message}) +@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskIDView(ApiBaseView): """resolves to /api/task-id// GET: return details of task id @@ -827,6 +844,7 @@ class TaskIDView(ApiBaseView): return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}" +@method_decorator(user_passes_test(check_admin), name="dispatch") class RefreshView(ApiBaseView): """resolves to /api/refresh/ GET: get refresh progress @@ -948,6 +966,7 @@ class SearchView(ApiBaseView): return Response(search_results) +@method_decorator(user_passes_test(check_admin), name="dispatch") class TokenView(ApiBaseView): """resolves to /api/token/ DELETE: revoke the token diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index 2adce0d3..b7b998ec 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -1,4 +1,5 @@ {% load static %} +{% load auth_extras %} @@ -57,9 +58,11 @@
playlists
 + {% if request.user|has_group:"admin" or request.user.is_staff %}
downloads
 + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel.html b/tubearchivist/home/templates/home/channel.html index 00913452..83a163d3 100644 --- a/tubearchivist/home/templates/home/channel.html +++ b/tubearchivist/home/templates/home/channel.html @@ -2,11 +2,13 @@ {% load static %} {% load humanize %} {% block content %} +{% load auth_extras %}

Channels

+ {% if request.user|has_group:"admin" or request.user.is_staff %}
add-icon
@@ -17,6 +19,7 @@
+ {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 838615b6..c3f535c9 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -2,6 +2,8 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %} +
channel_banner @@ -38,7 +40,9 @@

Subscribers: {{ channel_info.channel_subs|intcomma }}

{% endif %} {% if channel_info.channel_subscribed %} + {% if request.user|has_group:"admin" or request.user.is_staff %} + {% endif %} {% else %} {% endif %} diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 2a199d00..4122e4f4 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -53,7 +53,7 @@

{{ playlist.playlist_channel }}

{{ playlist.playlist_name }}

Last refreshed: {{ playlist.playlist_last_refresh }}

- {% if playlist.playlist_subscribed %} + {% if playlist.playlist_subscribed and request.user|has_group:"admin" or request.user.is_staff %} {% else %} diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index c6e02031..ab2d788c 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -1,11 +1,15 @@ {% extends "home/base.html" %} {% load static %} {% block content %} +{% load auth_extras %} +

Playlists

+ {% if request.user|has_group:"admin" or request.user.is_staff %} +
add-icon
@@ -16,6 +20,8 @@
+ {% endif %} +
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 3faf449b..5fbe1946 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -2,6 +2,8 @@ {% load static %} {% load humanize %} {% block content %} +{% load auth_extras %} +

{{ playlist_info.playlist_name }}

@@ -27,7 +29,9 @@

Last refreshed: {{ playlist_info.playlist_last_refresh }}

Playlist: {% if playlist_info.playlist_subscribed %} + {% if request.user|has_group:"admin" or request.user.is_staff %} + {% endif %} {% else %} {% endif %} diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index d1e43ae1..f4993656 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}

@@ -81,15 +82,19 @@ {% if reindex %}

Reindex scheduled

{% else %} + {% if request.user|has_group:"admin" or request.user.is_staff %}
+ {% endif %} {% endif %} + {% if request.user|has_group:"admin" or request.user.is_staff %}
Are you sure?
+ {% endif %}
diff --git a/tubearchivist/home/templatetags/__init__.py b/tubearchivist/home/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tubearchivist/home/templatetags/auth_extras.py b/tubearchivist/home/templatetags/auth_extras.py new file mode 100644 index 00000000..a41b4b1e --- /dev/null +++ b/tubearchivist/home/templatetags/auth_extras.py @@ -0,0 +1,8 @@ +from django import template + +register = template.Library() + + +@register.filter(name="has_group") +def has_group(user, group_name): + return user.groups.filter(name=group_name).exists() From 572b23169c62c837b21d24e1ef0ef47eab28d3e8 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 15 Oct 2023 14:56:54 +0700 Subject: [PATCH 03/13] finetune limited permission user --- tubearchivist/api/views.py | 10 +- .../home/templates/home/base_settings.html | 9 +- .../home/templates/home/channel_id.html | 4 +- .../home/templates/home/channel_id_about.html | 125 +++++++++--------- .../templates/home/channel_id_playlist.html | 5 +- tubearchivist/home/templates/home/video.html | 2 +- tubearchivist/home/views.py | 10 ++ 7 files changed, 96 insertions(+), 69 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 9d5eafa1..dc67e010 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -437,6 +437,7 @@ class PlaylistApiListView(ApiBaseView): self.get_document_list(request) return Response(self.response) + @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request): """subscribe/unsubscribe to list of playlists""" data = request.data @@ -483,6 +484,7 @@ class PlaylistApiView(ApiBaseView): self.get_document(playlist_id) return Response(self.response, status=self.status_code) + @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, playlist_id): """delete playlist""" print(f"{playlist_id}: delete playlist") @@ -513,6 +515,7 @@ class PlaylistApiVideoView(ApiBaseView): return Response(self.response, status=self.status_code) +@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadApiView(ApiBaseView): """resolves to /api/download// GET: returns metadata dict of an item in the download queue @@ -529,7 +532,6 @@ class DownloadApiView(ApiBaseView): self.get_document(video_id) return Response(self.response, status=self.status_code) - @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request, video_id): """post to video to change status""" item_status = request.data.get("status") @@ -550,7 +552,6 @@ class DownloadApiView(ApiBaseView): return Response(request.data) - @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def delete(request, video_id): # pylint: disable=unused-argument @@ -561,6 +562,7 @@ class DownloadApiView(ApiBaseView): return Response({"success": True}) +@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadApiListView(ApiBaseView): """resolves to /api/download/ GET: returns latest videos in the download queue @@ -596,7 +598,6 @@ class DownloadApiListView(ApiBaseView): self.get_document_list(request) return Response(self.response) - @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """add list of videos to download queue""" @@ -622,7 +623,6 @@ class DownloadApiListView(ApiBaseView): return Response(data) - @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request): """delete download queue""" query_filter = request.GET.get("filter", False) @@ -740,6 +740,7 @@ class SnapshotApiView(ApiBaseView): return Response(response) +@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskListView(ApiBaseView): """resolves to /api/task-name/ GET: return a list of all stored task results @@ -877,6 +878,7 @@ class RefreshView(ApiBaseView): return Response(data) +@method_decorator(user_passes_test(check_admin), name="dispatch") class CookieView(ApiBaseView): """resolves to /api/cookie/ GET: check if cookie is enabled diff --git a/tubearchivist/home/templates/home/base_settings.html b/tubearchivist/home/templates/home/base_settings.html index 3eb6c13b..31787ef5 100644 --- a/tubearchivist/home/templates/home/base_settings.html +++ b/tubearchivist/home/templates/home/base_settings.html @@ -1,14 +1,17 @@ {# Base file for all of the settings pages to ensure a common menu #} {% extends "home/base.html" %} {% load static %} +{% load auth_extras %} {% block content %}

Dashboard

User

 -

Application

 -

Scheduling

 -

Actions

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Application

 +

Scheduling

 +

Actions

 + {% endif %}
{% block settings_content %}{% endblock %} diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index c3f535c9..0d45bd87 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -21,7 +21,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index d4afcbc7..cd965f31 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}
channel_banner @@ -19,7 +20,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
@@ -56,19 +59,21 @@ {% elif channel_info.channel_views > 0 %}

Channel views: {{ channel_info.channel_views|intcomma }}

{% endif %} -
- -
- Delete {{ channel_info.channel_name }} including all videos? -
-
- {% if reindex %} -

Reindex scheduled

- {% else %} -
- - + {% if request.user|has_group:"admin" or request.user.is_staff %} +
+ +
+ Delete {{ channel_info.channel_name }} including all videos? +
+ {% if reindex %} +

Reindex scheduled

+ {% else %} +
+ + +
+ {% endif %} {% endif %}
@@ -90,53 +95,55 @@
{% endif %} -
-
-

Customize {{ channel_info.channel_name }}

-
- {% csrf_token %} -
-

Download format: - {% if channel_info.channel_overwrites.download_format %} - {{ channel_info.channel_overwrites.download_format }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.download_format }}
-
-
-

Auto delete watched videos after x days: - {% if channel_info.channel_overwrites.autodelete_days %} - {{ channel_info.channel_overwrites.autodelete_days }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.autodelete_days }}
-
-
-

Index playlists: - {% if channel_info.channel_overwrites.index_playlists %} - {{ channel_info.channel_overwrites.index_playlists }} - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.index_playlists }}
-
-
-

Enable SponsorBlock: - {% if channel_info.channel_overwrites.integrate_sponsorblock %} - {{ channel_info.channel_overwrites.integrate_sponsorblock }} - {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %} - Disabled - {% else %} - False - {% endif %}

- {{ channel_overwrite_form.integrate_sponsorblock }}
-
- -
+ {% if request.user|has_group:"admin" or request.user.is_staff %} +
+
+

Customize {{ channel_info.channel_name }}

+
+ {% csrf_token %} +
+

Download format: + {% if channel_info.channel_overwrites.download_format %} + {{ channel_info.channel_overwrites.download_format }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.download_format }}
+
+
+

Auto delete watched videos after x days: + {% if channel_info.channel_overwrites.autodelete_days %} + {{ channel_info.channel_overwrites.autodelete_days }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.autodelete_days }}
+
+
+

Index playlists: + {% if channel_info.channel_overwrites.index_playlists %} + {{ channel_info.channel_overwrites.index_playlists }} + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.index_playlists }}
+
+
+

Enable SponsorBlock: + {% if channel_info.channel_overwrites.integrate_sponsorblock %} + {{ channel_info.channel_overwrites.integrate_sponsorblock }} + {% elif channel_info.channel_overwrites.integrate_sponsorblock == False %} + Disabled + {% else %} + False + {% endif %}

+ {{ channel_overwrite_form.integrate_sponsorblock }}
+
+ +
+
-
+ {% endif %}
{% endblock content %} \ No newline at end of file diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 4122e4f4..539d4388 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -2,6 +2,7 @@ {% block content %} {% load static %} {% load humanize %} +{% load auth_extras %}
channel_banner @@ -19,7 +20,9 @@ {% endif %}

About

 {% if has_pending %} -

Downloads

 + {% if request.user|has_group:"admin" or request.user.is_staff %} +

Downloads

 + {% endif %} {% endif %}
diff --git a/tubearchivist/home/templates/home/video.html b/tubearchivist/home/templates/home/video.html index f4993656..b9ad04a1 100644 --- a/tubearchivist/home/templates/home/video.html +++ b/tubearchivist/home/templates/home/video.html @@ -88,8 +88,8 @@
{% endif %} {% endif %} - {% if request.user|has_group:"admin" or request.user.is_staff %} + {% if request.user|has_group:"admin" or request.user.is_staff %}
Are you sure? diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 488bef05..c3051bbb 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -9,11 +9,14 @@ import urllib.parse from time import sleep from api.src.search_processor import SearchProcess, process_aggs +from api.views import check_admin from django.conf import settings from django.contrib.auth import login +from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.forms import AuthenticationForm from django.http import Http404, JsonResponse from django.shortcuts import redirect, render +from django.utils.decorators import method_decorator from django.views import View from home.src.download.queue import PendingInteract from home.src.download.yt_dlp_base import CookieHandler @@ -317,6 +320,7 @@ class AboutView(MinView): return render(request, "home/about.html", context) +@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadView(ArchivistResultsView): """resolves to /download/ handle the download queue @@ -597,6 +601,7 @@ class ChannelIdAboutView(ChannelIdBaseView): return render(request, "home/channel_id_about.html", self.context) + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request, channel_id): """handle post request""" @@ -681,6 +686,7 @@ class ChannelView(ArchivistResultsView): "term": {"channel_subscribed": {"value": True}} } + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """handle http post requests""" @@ -824,6 +830,7 @@ class PlaylistView(ArchivistResultsView): } } + @method_decorator(user_passes_test(check_admin), name="dispatch") @staticmethod def post(request): """handle post from search form""" @@ -986,6 +993,7 @@ class SettingsUserView(MinView): return redirect("settings_user", permanent=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsApplicationView(MinView): """resolves to /settings/application/ handle the settings sub-page for application configuration, @@ -1075,6 +1083,7 @@ class SettingsApplicationView(MinView): RedisArchivist().set_message(key, message=message, expire=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsSchedulingView(MinView): """resolves to /settings/scheduling/ handle the settings sub-page for scheduling settings, @@ -1108,6 +1117,7 @@ class SettingsSchedulingView(MinView): return redirect("settings_scheduling", permanent=True) +@method_decorator(user_passes_test(check_admin), name="dispatch") class SettingsActionsView(MinView): """resolves to /settings/actions/ handle the settings actions sub-page From 5165c3e34a0b7ced88a2e6f3dbd88b19f4a6048f Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 16 Oct 2023 16:12:28 +0700 Subject: [PATCH 04/13] bump requirements --- tubearchivist/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index 11ab42d9..e5a5daa2 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,13 +1,13 @@ -apprise==1.5.0 +apprise==1.6.0 celery==5.3.4 Django==4.2.6 django-auth-ldap==4.6.0 -django-cors-headers==4.2.0 +django-cors-headers==4.3.0 djangorestframework==3.14.0 -Pillow==10.0.1 +Pillow==10.1.0 redis==5.0.1 requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.22 -whitenoise==6.5.0 -yt-dlp==2023.10.7 +whitenoise==6.6.0 +yt-dlp==2023.10.13 From 4d111aff826f6aeed03522c7a44b632f699aa88a Mon Sep 17 00:00:00 2001 From: Clark <104835586+anonamouslyginger@users.noreply.github.com> Date: Sat, 28 Oct 2023 03:27:03 +0000 Subject: [PATCH 05/13] Move the startup application settings to a new class (#571) * Move the startup application settings to a new class * Replace settings methods with static fields * Move Redis and ES configuration to the settings class * Fix environment python imports * Update envcheck to use the new settings --- tubearchivist/api/src/search_processor.py | 5 +-- tubearchivist/api/views.py | 9 ++++- .../config/management/commands/ta_envcheck.py | 19 ++++----- .../config/management/commands/ta_migpath.py | 9 ++--- .../config/management/commands/ta_startup.py | 8 ++-- tubearchivist/config/settings.py | 8 ++-- tubearchivist/home/config.json | 4 -- tubearchivist/home/src/download/thumbnails.py | 12 +++--- .../home/src/download/yt_dlp_base.py | 7 +++- .../home/src/download/yt_dlp_handler.py | 24 ++++++------ tubearchivist/home/src/es/backup.py | 7 ++-- tubearchivist/home/src/es/connect.py | 24 ++++++------ tubearchivist/home/src/es/snapshot.py | 3 +- tubearchivist/home/src/index/channel.py | 5 ++- tubearchivist/home/src/index/filesystem.py | 4 +- tubearchivist/home/src/index/generic.py | 1 - tubearchivist/home/src/index/manual.py | 11 +++--- tubearchivist/home/src/index/reindex.py | 5 ++- tubearchivist/home/src/index/subtitle.py | 9 +++-- tubearchivist/home/src/index/video.py | 7 ++-- tubearchivist/home/src/ta/config.py | 19 --------- tubearchivist/home/src/ta/helper.py | 8 ++-- tubearchivist/home/src/ta/settings.py | 39 +++++++++++++++++++ tubearchivist/home/src/ta/ta_redis.py | 11 +++--- tubearchivist/home/src/ta/users.py | 7 +--- tubearchivist/home/tasks.py | 11 ++++-- tubearchivist/home/views.py | 7 ++-- tubearchivist/static/script.js | 2 +- 28 files changed, 153 insertions(+), 132 deletions(-) create mode 100644 tubearchivist/home/src/ta/settings.py diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 232474d4..9f77b05e 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -7,15 +7,14 @@ Functionality: import urllib.parse from home.src.download.thumbnails import ThumbManager -from home.src.ta.config import AppConfig from home.src.ta.helper import date_praser, get_duration_str +from home.src.ta.settings import EnvironmentSettings class SearchProcess: """process search results""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR def __init__(self, response): self.response = response diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index dc67e010..54c88fba 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -20,6 +20,7 @@ from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig, ReleaseVersion +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser @@ -56,7 +57,13 @@ class ApiBaseView(APIView): def __init__(self): super().__init__() - self.response = {"data": False, "config": AppConfig().config} + self.response = { + "data": False, + "config": { + "enable_cast": EnvironmentSettings.ENABLE_CAST, + "downloads": AppConfig().config["downloads"], + }, + } self.data = {"query": {"match_all": {}}} self.status_code = False self.context = False diff --git a/tubearchivist/config/management/commands/ta_envcheck.py b/tubearchivist/config/management/commands/ta_envcheck.py index 446962d9..76c5ba10 100644 --- a/tubearchivist/config/management/commands/ta_envcheck.py +++ b/tubearchivist/config/management/commands/ta_envcheck.py @@ -11,6 +11,7 @@ import re from django.core.management.base import BaseCommand, CommandError from home.models import Account +from home.src.ta.settings import EnvironmentSettings LOGO = """ @@ -96,18 +97,14 @@ class Command(BaseCommand): def _elastic_user_overwrite(self): """check for ELASTIC_USER overwrite""" - self.stdout.write("[2] set default ES user") - if not os.environ.get("ELASTIC_USER"): - os.environ.setdefault("ELASTIC_USER", "elastic") - - env = os.environ.get("ELASTIC_USER") - + self.stdout.write("[2] check ES user overwrite") + env = EnvironmentSettings.ES_USER self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}")) def _ta_port_overwrite(self): """set TA_PORT overwrite for nginx""" self.stdout.write("[3] check TA_PORT overwrite") - overwrite = os.environ.get("TA_PORT") + overwrite = EnvironmentSettings.TA_PORT if not overwrite: self.stdout.write(self.style.SUCCESS(" TA_PORT is not set")) return @@ -125,7 +122,7 @@ class Command(BaseCommand): def _ta_uwsgi_overwrite(self): """set TA_UWSGI_PORT overwrite""" self.stdout.write("[4] check TA_UWSGI_PORT overwrite") - overwrite = os.environ.get("TA_UWSGI_PORT") + overwrite = EnvironmentSettings.TA_UWSGI_PORT if not overwrite: message = " TA_UWSGI_PORT is not set" self.stdout.write(self.style.SUCCESS(message)) @@ -151,7 +148,7 @@ class Command(BaseCommand): def _enable_cast_overwrite(self): """cast workaround, remove auth for static files in nginx""" self.stdout.write("[5] check ENABLE_CAST overwrite") - overwrite = os.environ.get("ENABLE_CAST") + overwrite = EnvironmentSettings.ENABLE_CAST if not overwrite: self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set")) return @@ -174,8 +171,8 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS(message)) return - name = os.environ.get("TA_USERNAME") - password = os.environ.get("TA_PASSWORD") + name = EnvironmentSettings.TA_USERNAME + password = EnvironmentSettings.TA_PASSWORD Account.objects.create_superuser(name, password) message = f" ✓ new superuser with name {name} created" self.stdout.write(self.style.SUCCESS(message)) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 607c58ee..05f0b76f 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -6,8 +6,8 @@ import shutil from django.core.management.base import BaseCommand from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings TOPIC = """ @@ -58,8 +58,7 @@ class FolderMigration: """migrate video archive folder""" def __init__(self): - self.config = AppConfig().config - self.videos = self.config["application"]["videos"] + self.videos = EnvironmentSettings.MEDIA_DIR self.bulk_list = [] def get_to_migrate(self): @@ -84,8 +83,8 @@ class FolderMigration: def create_folders(self, to_migrate): """create required channel folders""" - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} for channel_id in all_channel_ids: diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 602731f8..347e4385 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -14,6 +14,7 @@ from home.src.es.snapshot import ElasticSnapshot from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import clear_dl_cache +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.users import UserConfig @@ -69,7 +70,7 @@ class Command(BaseCommand): "playlists", "videos", ] - cache_dir = AppConfig().config["application"]["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR for folder in folders: folder_path = os.path.join(cache_dir, folder) os.makedirs(folder_path, exist_ok=True) @@ -119,8 +120,7 @@ class Command(BaseCommand): def _clear_dl_cache(self): """clear leftover files from dl cache""" self.stdout.write("[5] clear leftover files from dl cache") - config = AppConfig().config - leftover_files = clear_dl_cache(config) + leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR) if leftover_files: self.stdout.write( self.style.SUCCESS(f" ✓ cleared {leftover_files} files") @@ -152,7 +152,7 @@ class Command(BaseCommand): def _mig_set_streams(self): """migration: update from 0.3.5 to 0.3.6, set streams and media_size""" self.stdout.write("[MIGRATION] index streams and media size") - videos = AppConfig().config["application"]["videos"] + videos = EnvironmentSettings.MEDIA_DIR data = { "query": { "bool": {"must_not": [{"exists": {"field": "streams"}}]} diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 01759857..f4106551 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -17,8 +17,8 @@ from pathlib import Path import ldap from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch -from home.src.ta.config import AppConfig from home.src.ta.helper import ta_host_parser +from home.src.ta.settings import EnvironmentSettings # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ -PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) +PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode()) SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! @@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")): # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases -CACHE_DIR = AppConfig().config["application"]["cache_dir"] +CACHE_DIR = EnvironmentSettings.CACHE_DIR DB_PATH = path.join(CACHE_DIR, "db.sqlite3") DATABASES = { "default": { @@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): # https://docs.djangoproject.com/en/3.2/topics/i18n/ LANGUAGE_CODE = "en-us" -TIME_ZONE = environ.get("TZ") or "UTC" +TIME_ZONE = EnvironmentSettings.TZ USE_I18N = True USE_L10N = True USE_TZ = True diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index 26d3bf90..1b655f47 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -25,10 +25,6 @@ "integrate_sponsorblock": false }, "application": { - "app_root": "/app", - "cache_dir": "/cache", - "videos": "/youtube", - "enable_cast": false, "enable_snapshot": true }, "scheduler": { diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index ca498c58..896f6031 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -11,7 +11,7 @@ from time import sleep import requests from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from mutagen.mp4 import MP4, MP4Cover from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError @@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True class ThumbManagerBase: """base class for thumbnail management""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR VIDEO_DIR = os.path.join(CACHE_DIR, "videos") CHANNEL_DIR = os.path.join(CACHE_DIR, "channels") PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists") @@ -70,7 +69,7 @@ class ThumbManagerBase: img_raw = Image.open(self.fallback) return img_raw - app_root = self.CONFIG["application"]["app_root"] + app_root = EnvironmentSettings.APP_DIR default_map = { "video": os.path.join( app_root, "static/img/default-video-thumb.jpg" @@ -380,9 +379,8 @@ class ThumbFilesystem: class EmbedCallback: """callback class to embed thumbnails""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] - MEDIA_DIR = CONFIG["application"]["videos"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR + MEDIA_DIR = EnvironmentSettings.MEDIA_DIR FORMAT = MP4Cover.FORMAT_JPEG def __init__(self, source, index_name, counter=0): diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index b39601cd..b1c8fa75 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -10,6 +10,7 @@ from http import cookiejar from io import StringIO import yt_dlp +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist @@ -86,6 +87,7 @@ class CookieHandler: def __init__(self, config): self.cookie_io = False self.config = config + self.cache_dir = EnvironmentSettings.CACHE_DIR def get(self): """get cookie io stream""" @@ -95,8 +97,9 @@ class CookieHandler: def import_cookie(self): """import cookie from file""" - cache_path = self.config["application"]["cache_dir"] - import_path = os.path.join(cache_path, "import", "cookies.google.txt") + import_path = os.path.join( + self.cache_dir, "import", "cookies.google.txt" + ) try: with open(import_path, encoding="utf-8") as cookie_file: diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 9a865aba..efcdb9b4 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class DownloadPostProcess: @@ -153,6 +154,8 @@ class VideoDownloader: self.youtube_id_list = youtube_id_list self.task = task self.config = AppConfig().config + self.cache_dir = EnvironmentSettings.CACHE_DIR + self.media_dir = EnvironmentSettings.MEDIA_DIR self._build_obs() self.channels = set() self.videos = set() @@ -262,10 +265,7 @@ class VideoDownloader: """initial obs""" self.obs = { "merge_output_format": "mp4", - "outtmpl": ( - self.config["application"]["cache_dir"] - + "/download/%(id)s.mp4" - ), + "outtmpl": (self.cache_dir + "/download/%(id)s.mp4"), "progress_hooks": [self._progress_hook], "noprogress": True, "continuedl": True, @@ -340,7 +340,7 @@ class VideoDownloader: if format_overwrite: obs["format"] = format_overwrite - dl_cache = self.config["application"]["cache_dir"] + "/download/" + dl_cache = self.cache_dir + "/download/" # check if already in cache to continue from there all_cached = ignore_filelist(os.listdir(dl_cache)) @@ -370,20 +370,20 @@ class VideoDownloader: def move_to_archive(self, vid_dict): """move downloaded video from cache to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID # make folder - folder = os.path.join(videos, vid_dict["channel"]["channel_id"]) + folder = os.path.join( + self.media_dir, vid_dict["channel"]["channel_id"] + ) if not os.path.exists(folder): os.makedirs(folder) if host_uid and host_gid: os.chown(folder, host_uid, host_gid) # move media file media_file = vid_dict["youtube_id"] + ".mp4" - cache_dir = self.config["application"]["cache_dir"] - old_path = os.path.join(cache_dir, "download", media_file) - new_path = os.path.join(videos, vid_dict["media_url"]) + old_path = os.path.join(self.cache_dir, "download", media_file) + new_path = os.path.join(self.media_dir, vid_dict["media_url"]) # move media file and fix permission shutil.move(old_path, new_path, copy_function=shutil.copyfile) if host_uid and host_gid: diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index 3dc1cf5a..b23592a4 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -13,6 +13,7 @@ from datetime import datetime from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import get_mapping, ignore_filelist +from home.src.ta.settings import EnvironmentSettings class ElasticBackup: @@ -22,7 +23,7 @@ class ElasticBackup: def __init__(self, reason=False, task=False): self.config = AppConfig().config - self.cache_dir = self.config["application"]["cache_dir"] + self.cache_dir = EnvironmentSettings.CACHE_DIR self.timestamp = datetime.now().strftime("%Y%m%d") self.index_config = get_mapping() self.reason = reason @@ -217,6 +218,7 @@ class BackupCallback: self.index_name = index_name self.counter = counter self.timestamp = datetime.now().strftime("%Y%m%d") + self.cache_dir = EnvironmentSettings.CACHE_DIR def run(self): """run the junk task""" @@ -243,9 +245,8 @@ class BackupCallback: def _write_es_json(self, file_content): """write nd-json file for es _bulk API to disk""" - cache_dir = AppConfig().config["application"]["cache_dir"] index = self.index_name.lstrip("ta_") file_name = f"es_{index}-{self.timestamp}-{self.counter}.json" - file_path = os.path.join(cache_dir, "backup", file_name) + file_path = os.path.join(self.cache_dir, "backup", file_name) with open(file_path, "a+", encoding="utf-8") as f: f.write(file_content) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index a7c3ff5f..f343b8e4 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -6,11 +6,11 @@ functionality: # pylint: disable=missing-timeout import json -import os from typing import Any import requests import urllib3 +from home.src.ta.settings import EnvironmentSettings class ElasticWrap: @@ -18,16 +18,14 @@ class ElasticWrap: returns response json and status code tuple """ - ES_URL: str = str(os.environ.get("ES_URL")) - ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) - ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic") - ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) - def __init__(self, path: str): - self.url: str = f"{self.ES_URL}/{path}" - self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS) + self.url: str = f"{EnvironmentSettings.ES_URL}/{path}" + self.auth: tuple[str, str] = ( + EnvironmentSettings.ES_USER, + EnvironmentSettings.ES_PASS, + ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def get( @@ -43,7 +41,7 @@ class ElasticWrap: "timeout": timeout, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False if data: @@ -78,7 +76,7 @@ class ElasticWrap: } ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.post(self.url, **kwargs) @@ -103,7 +101,7 @@ class ElasticWrap: "auth": self.auth, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.put(self.url, **kwargs) @@ -130,7 +128,7 @@ class ElasticWrap: if data: kwargs["json"] = data - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.delete(self.url, **kwargs) diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index 15fc82c6..d77f0761 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo from home.src.es.connect import ElasticWrap from home.src.ta.helper import get_mapping +from home.src.ta.settings import EnvironmentSettings class ElasticSnapshot: @@ -256,7 +257,7 @@ class ElasticSnapshot: expected_format = "%Y-%m-%dT%H:%M:%S.%fZ" date = datetime.strptime(date_utc, expected_format) local_datetime = date.replace(tzinfo=ZoneInfo("localtime")) - converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ"))) + converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ)) converted_str = converted.strftime("%Y-%m-%d %H:%M") return converted_str diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 19070626..02e57266 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.index.playlist import YoutubePlaylist +from home.src.ta.settings import EnvironmentSettings class YoutubeChannel(YouTubeItem): @@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem): def _info_json_fallback(self): """read channel info.json for additional metadata""" info_json = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{self.youtube_id}.info.json", ) @@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem): def get_folder_path(self): """get folder where media files get stored""" folder_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel_id"], ) return folder_path diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index ecd35092..ab208c22 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -8,14 +8,14 @@ import os from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo, index_new_video -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class Scanner: """scan index and filesystem""" - VIDEOS: str = AppConfig().config["application"]["videos"] + VIDEOS: str = EnvironmentSettings.MEDIA_DIR def __init__(self, task=False) -> None: self.task = task diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index a5f624de..e18623ee 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -26,7 +26,6 @@ class YouTubeItem: self.youtube_id = youtube_id self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.config = AppConfig().config - self.app_conf = self.config["application"] self.youtube_meta = False self.json_data = False diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index 3a19588f..a25b0611 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -16,6 +16,7 @@ from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings from PIL import Image from yt_dlp.utils import ISO639Utils @@ -28,7 +29,7 @@ class ImportFolderScanner: """ CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR IMPORT_DIR = os.path.join(CACHE_DIR, "import") """All extensions should be in lowercase until better handling is in place. @@ -433,9 +434,9 @@ class ManualImport: def _move_to_archive(self, json_data): """move identified media file to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + videos = EnvironmentSettings.MEDIA_DIR + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID channel, file = os.path.split(json_data["media_url"]) channel_folder = os.path.join(videos, channel) @@ -472,7 +473,7 @@ class ManualImport: os.remove(subtitle_file) channel_info = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{json_data['channel']['channel_id']}.info.json", ) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b89c00d9..c25115c4 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -19,6 +19,7 @@ from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisQueue @@ -293,7 +294,7 @@ class Reindex(ReindexBase): # get new media_url = os.path.join( - self.config["application"]["videos"], es_meta["media_url"] + EnvironmentSettings.MEDIA_DIR, es_meta["media_url"] ) video.build_json(media_path=media_url) if not video.youtube_meta: @@ -328,7 +329,7 @@ class Reindex(ReindexBase): def _rename_media_file(self, media_url_is, media_url_should): """handle title change""" print(f"[reindex] fix media_url {media_url_is} to {media_url_should}") - videos = self.config["application"]["videos"] + videos = EnvironmentSettings.MEDIA_DIR old_path = os.path.join(videos, media_url_is) new_path = os.path.join(videos, media_url_should) os.rename(old_path, new_path) diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index 18af56ac..b57ad55b 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -12,6 +12,7 @@ from datetime import datetime import requests from home.src.es.connect import ElasticWrap from home.src.ta.helper import requests_headers +from home.src.ta.settings import EnvironmentSettings class YoutubeSubtitle: @@ -113,7 +114,7 @@ class YoutubeSubtitle: def download_subtitles(self, relevant_subtitles): """download subtitle files to archive""" - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR indexed = [] for subtitle in relevant_subtitles: dest_path = os.path.join(videos_base, subtitle["media_url"]) @@ -149,8 +150,8 @@ class YoutubeSubtitle: with open(dest_path, "w", encoding="utf-8") as subfile: subfile.write(subtitle_str) - host_uid = self.video.config["application"]["HOST_UID"] - host_gid = self.video.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID if host_uid and host_gid: os.chown(dest_path, host_uid, host_gid) @@ -162,7 +163,7 @@ class YoutubeSubtitle: def delete(self, subtitles=False): """delete subtitles from index and filesystem""" youtube_id = self.video.youtube_id - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR # delete files if subtitles: files = [i["media_url"] for i in subtitles] diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 606e32f3..b4143506 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor +from home.src.ta.settings import EnvironmentSettings from home.src.ta.users import UserConfig from ryd_client import ryd_client @@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def build_dl_cache_path(self): """find video path in dl cache""" - cache_dir = self.app_conf["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR video_id = self.json_data["youtube_id"] cache_path = f"{cache_dir}/download/{video_id}.mp4" if os.path.exists(cache_path): return cache_path channel_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel"]["channel_id"], f"{video_id}.mp4", ) @@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if not self.json_data: raise FileNotFoundError - video_base = self.app_conf["videos"] + video_base = EnvironmentSettings.MEDIA_DIR media_url = self.json_data.get("media_url") file_path = os.path.join(video_base, media_url) try: diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index a32d0830..c6e9e68f 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -5,7 +5,6 @@ Functionality: """ import json -import os import re from random import randint from time import sleep @@ -28,7 +27,6 @@ class AppConfig: if not config: config = self.get_config_file() - config["application"].update(self.get_config_env()) return config def get_config_file(self): @@ -36,25 +34,8 @@ class AppConfig: with open("home/config.json", "r", encoding="utf-8") as f: config_file = json.load(f) - config_file["application"].update(self.get_config_env()) - return config_file - @staticmethod - def get_config_env(): - """read environment application variables. - - Connection to ES is managed in ElasticWrap and the - connection to Redis is managed in RedisArchivist.""" - - application = { - "HOST_UID": int(os.environ.get("HOST_UID", False)), - "HOST_GID": int(os.environ.get("HOST_GID", False)), - "enable_cast": bool(os.environ.get("ENABLE_CAST")), - } - - return application - @staticmethod def get_config_redis(): """read config json set from redis to overwrite defaults""" diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index db6e4b6c..4143ed6e 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float: return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds) -def clear_dl_cache(config: dict) -> int: +def clear_dl_cache(cache_dir: str) -> int: """clear leftover files from dl cache""" print("clear download cache") - cache_dir = os.path.join(config["application"]["cache_dir"], "download") - leftover_files = ignore_filelist(os.listdir(cache_dir)) + download_cache_dir = os.path.join(cache_dir, "download") + leftover_files = ignore_filelist(os.listdir(download_cache_dir)) for cached in leftover_files: - to_delete = os.path.join(cache_dir, cached) + to_delete = os.path.join(download_cache_dir, cached) os.remove(to_delete) return len(leftover_files) diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py new file mode 100644 index 00000000..0798107e --- /dev/null +++ b/tubearchivist/home/src/ta/settings.py @@ -0,0 +1,39 @@ +""" +Functionality: +- read and write application config backed by ES +- encapsulate persistence of application properties +""" +import os + + +class EnvironmentSettings: + """ + Handle settings for the application that are driven from the environment. + These will not change when the user is using the application. + These settings are only provided only on startup. + """ + + HOST_UID: int = int(os.environ.get("HOST_UID", False)) + HOST_GID: int = int(os.environ.get("HOST_GID", False)) + ENABLE_CAST: bool = bool(os.environ.get("ENABLE_CAST")) + TZ: str = str(os.environ.get("TZ", "UTC")) + TA_PORT: int = int(os.environ.get("TA_PORT", False)) + TA_UWSGI_PORT: int = int(os.environ.get("TA_UWSGI_PORT", False)) + TA_USERNAME: str = str(os.environ.get("TA_USERNAME")) + TA_PASSWORD: str = str(os.environ.get("TA_PASSWORD")) + + # Application Paths + MEDIA_DIR: str = str(os.environ.get("TA_MEDIA_DIR", "/youtube")) + APP_DIR: str = str(os.environ.get("TA_APP_DIR", "/app")) + CACHE_DIR: str = str(os.environ.get("TA_CACHE_DIR", "/cache")) + + # Redis + REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) + REDIS_PORT: int = int(os.environ.get("REDIS_PORT", 6379)) + REDIS_NAME_SPACE: str = str(os.environ.get("REDIS_NAME_SPACE", "ta:")) + + # ElasticSearch + ES_URL: str = str(os.environ.get("ES_URL")) + ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) + ES_USER: str = str(os.environ.get("ELASTIC_USER", "elastic")) + ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 77de5287..de821fbc 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -6,20 +6,21 @@ functionality: """ import json -import os import redis +from home.src.ta.settings import EnvironmentSettings class RedisBase: """connection base for redis""" - REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) - REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379) - NAME_SPACE: str = "ta:" + NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE def __init__(self): - self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) + self.conn = redis.Redis( + host=EnvironmentSettings.REDIS_HOST, + port=EnvironmentSettings.REDIS_PORT, + ) class RedisArchivist(RedisBase): diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py index c337381f..57b342c3 100644 --- a/tubearchivist/home/src/ta/users.py +++ b/tubearchivist/home/src/ta/users.py @@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False): class UserConfig: - """Handle settings for an individual user - - Create getters and setters for usage in the application. - Although tedious it helps prevents everything caring about how properties - are persisted. Plus it allows us to save anytime any value is set. - """ + """Handle settings for an individual user""" _DEFAULT_USER_SETTINGS = UserConfigType( colors="dark", diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index a2c4cdbf..88b09695 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.notify import Notifications +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.urlparser import Parser CONFIG = AppConfig().config -REDIS_HOST = os.environ.get("REDIS_HOST") -REDIS_PORT = os.environ.get("REDIS_PORT") or 6379 +REDIS_HOST = EnvironmentSettings.REDIS_HOST +REDIS_PORT = EnvironmentSettings.REDIS_PORT os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") app = Celery( @@ -39,9 +40,11 @@ app = Celery( backend=f"redis://{REDIS_HOST}:{REDIS_PORT}", result_extended=True, ) -app.config_from_object("django.conf:settings", namespace="ta:") +app.config_from_object( + "django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE +) app.autodiscover_tasks() -app.conf.timezone = os.environ.get("TZ") or "UTC" +app.conf.timezone = EnvironmentSettings.TZ class BaseTask(Task): diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index c3051bbb..dae3d175 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -42,6 +42,7 @@ from home.src.index.reindex import ReindexProgress from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.helper import time_parser +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.users import UserConfig from home.tasks import index_channel_playlists, subscribe_to @@ -56,7 +57,6 @@ class ArchivistViewConfig(View): self.view_origin = view_origin self.user_id = False self.user_conf: UserConfig = False - self.default_conf = False self.context = False def get_all_view_styles(self): @@ -73,11 +73,10 @@ class ArchivistViewConfig(View): """build default context for every view""" self.user_id = user_id self.user_conf = UserConfig(self.user_id) - self.default_conf = AppConfig().config self.context = { "colors": self.user_conf.get_value("colors"), - "cast": self.default_conf["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "sort_by": self.user_conf.get_value("sort_by"), "sort_order": self.user_conf.get_value("sort_order"), "view_style": self.user_conf.get_value( @@ -877,7 +876,7 @@ class VideoView(MinView): "video": video_data, "playlist_nav": playlist_nav, "title": video_data.get("title"), - "cast": config_handler.config["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "config": config_handler.config, "position": time_parser(request.GET.get("t")), "reindex": reindex.get("state"), diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index e6d3b862..cb40b268 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -470,7 +470,7 @@ function createPlayer(button) { // If cast integration is enabled create cast button let castButton = ''; - if (videoData.config.application.enable_cast) { + if (videoData.config.enable_cast) { castButton = ``; } From 8eaed07cff19f6cff4bf41b0a4568d370315c8da Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 28 Oct 2023 10:29:10 +0700 Subject: [PATCH 06/13] remove unused renamer --- tubearchivist/home/src/index/reindex.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index c25115c4..e87b30aa 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -312,10 +312,6 @@ class Reindex(ReindexBase): video.json_data["playlist"] = es_meta.get("playlist") video.upload_to_es() - if es_meta.get("media_url") != video.json_data["media_url"]: - self._rename_media_file( - es_meta.get("media_url"), video.json_data["media_url"] - ) thumb_handler = ThumbManager(youtube_id) thumb_handler.delete_video_thumb() @@ -326,14 +322,6 @@ class Reindex(ReindexBase): return - def _rename_media_file(self, media_url_is, media_url_should): - """handle title change""" - print(f"[reindex] fix media_url {media_url_is} to {media_url_should}") - videos = EnvironmentSettings.MEDIA_DIR - old_path = os.path.join(videos, media_url_is) - new_path = os.path.join(videos, media_url_should) - os.rename(old_path, new_path) - def _reindex_single_channel(self, channel_id): """refresh channel data and sync to videos""" # read current state From ea9ed6c238d84d92adaaf5a2031a15b4881eba4e Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 28 Oct 2023 10:30:21 +0700 Subject: [PATCH 07/13] fix linter --- tubearchivist/config/management/commands/ta_startup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 347e4385..e10cd1aa 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -299,12 +299,12 @@ class Command(BaseCommand): f" ✓ Settings for user '{user}' migrated to ES" ) ) - except Exception as e: + except Exception as err: message = " 🗙 user migration to ES failed" self.stdout.write(self.style.ERROR(message)) - self.stdout.write(self.style.ERROR(e)) + self.stdout.write(self.style.ERROR(err)) sleep(60) - raise CommandError(message) + raise CommandError(message) from err else: self.stdout.write( self.style.SUCCESS( From 21fde5e068d3b05eeda0905b4b83d30883583827 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 28 Oct 2023 15:03:16 +0700 Subject: [PATCH 08/13] remove old migrations --- .../config/management/commands/ta_startup.py | 77 ------------------- 1 file changed, 77 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index e10cd1aa..b10fc997 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -8,10 +8,8 @@ import os from time import sleep from django.core.management.base import BaseCommand, CommandError -from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.index_setup import ElasitIndexWrap from home.src.es.snapshot import ElasticSnapshot -from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import clear_dl_cache from home.src.ta.settings import EnvironmentSettings @@ -44,8 +42,6 @@ class Command(BaseCommand): self._version_check() self._mig_index_setup() self._mig_snapshot_check() - self._mig_set_streams() - self._mig_set_autostart() self._mig_move_users_to_es() def _sync_redis_state(self): @@ -149,79 +145,6 @@ class Command(BaseCommand): self.stdout.write("[MIGRATION] setup snapshots") ElasticSnapshot().setup() - def _mig_set_streams(self): - """migration: update from 0.3.5 to 0.3.6, set streams and media_size""" - self.stdout.write("[MIGRATION] index streams and media size") - videos = EnvironmentSettings.MEDIA_DIR - data = { - "query": { - "bool": {"must_not": [{"exists": {"field": "streams"}}]} - }, - "_source": ["media_url", "youtube_id"], - } - all_missing = IndexPaginate("ta_video", data).get_results() - if not all_missing: - self.stdout.write(" no videos need updating") - return - - total = len(all_missing) - for idx, missing in enumerate(all_missing): - media_url = missing["media_url"] - youtube_id = missing["youtube_id"] - media_path = os.path.join(videos, media_url) - if not os.path.exists(media_path): - self.stdout.write(f" file not found: {media_path}") - self.stdout.write(" run file system rescan to fix") - continue - - media = MediaStreamExtractor(media_path) - vid_data = { - "doc": { - "streams": media.extract_metadata(), - "media_size": media.get_file_size(), - } - } - path = f"ta_video/_update/{youtube_id}" - response, status_code = ElasticWrap(path).post(data=vid_data) - if not status_code == 200: - self.stdout.errors( - f" update failed: {path}, {response}, {status_code}" - ) - - if idx % 100 == 0: - self.stdout.write(f" progress {idx}/{total}") - - def _mig_set_autostart(self): - """migration: update from 0.3.5 to 0.3.6 set auto_start to false""" - self.stdout.write("[MIGRATION] set default download auto_start") - data = { - "query": { - "bool": {"must_not": [{"exists": {"field": "auto_start"}}]} - }, - "script": {"source": "ctx._source['auto_start'] = false"}, - } - path = "ta_download/_update_by_query" - response, status_code = ElasticWrap(path).post(data=data) - if status_code == 200: - updated = response.get("updated", 0) - if updated: - self.stdout.write( - self.style.SUCCESS( - f" ✓ {updated} videos updated in ta_download" - ) - ) - else: - self.stdout.write( - " no videos needed updating in ta_download" - ) - return - - message = " 🗙 ta_download auto_start update failed" - self.stdout.write(self.style.ERROR(message)) - self.stdout.write(response) - sleep(60) - raise CommandError(message) - def _mig_move_users_to_es(self): # noqa: C901 """migration: update from 0.4.1 to 0.4.2 move user config to ES""" self.stdout.write("[MIGRATION] move user configuration to ES") From 64ffc18da70dc2c9171adb9840aa80ad6b0acabe Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 28 Oct 2023 15:16:22 +0700 Subject: [PATCH 09/13] add debug methods for EnvironmentSettings --- tubearchivist/home/src/ta/settings.py | 88 +++++++++++++++++++++------ 1 file changed, 69 insertions(+), 19 deletions(-) diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py index 0798107e..db609c76 100644 --- a/tubearchivist/home/src/ta/settings.py +++ b/tubearchivist/home/src/ta/settings.py @@ -3,7 +3,8 @@ Functionality: - read and write application config backed by ES - encapsulate persistence of application properties """ -import os + +from os import environ class EnvironmentSettings: @@ -13,27 +14,76 @@ class EnvironmentSettings: These settings are only provided only on startup. """ - HOST_UID: int = int(os.environ.get("HOST_UID", False)) - HOST_GID: int = int(os.environ.get("HOST_GID", False)) - ENABLE_CAST: bool = bool(os.environ.get("ENABLE_CAST")) - TZ: str = str(os.environ.get("TZ", "UTC")) - TA_PORT: int = int(os.environ.get("TA_PORT", False)) - TA_UWSGI_PORT: int = int(os.environ.get("TA_UWSGI_PORT", False)) - TA_USERNAME: str = str(os.environ.get("TA_USERNAME")) - TA_PASSWORD: str = str(os.environ.get("TA_PASSWORD")) + HOST_UID: int = int(environ.get("HOST_UID", False)) + HOST_GID: int = int(environ.get("HOST_GID", False)) + ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST")) + TZ: str = str(environ.get("TZ", "UTC")) + TA_PORT: int = int(environ.get("TA_PORT", False)) + TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False)) + TA_USERNAME: str = str(environ.get("TA_USERNAME")) + TA_PASSWORD: str = str(environ.get("TA_PASSWORD")) # Application Paths - MEDIA_DIR: str = str(os.environ.get("TA_MEDIA_DIR", "/youtube")) - APP_DIR: str = str(os.environ.get("TA_APP_DIR", "/app")) - CACHE_DIR: str = str(os.environ.get("TA_CACHE_DIR", "/cache")) + MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube")) + APP_DIR: str = str(environ.get("TA_APP_DIR", "/app")) + CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache")) # Redis - REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) - REDIS_PORT: int = int(os.environ.get("REDIS_PORT", 6379)) - REDIS_NAME_SPACE: str = str(os.environ.get("REDIS_NAME_SPACE", "ta:")) + REDIS_HOST: str = str(environ.get("REDIS_HOST")) + REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379)) + REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:")) # ElasticSearch - ES_URL: str = str(os.environ.get("ES_URL")) - ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) - ES_USER: str = str(os.environ.get("ELASTIC_USER", "elastic")) - ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) + ES_URL: str = str(environ.get("ES_URL")) + ES_PASS: str = str(environ.get("ELASTIC_PASSWORD")) + ES_USER: str = str(environ.get("ELASTIC_USER", "elastic")) + ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL")) + + def print_generic(self): + """print generic env vars""" + print( + f""" + HOST_UID: {self.HOST_UID} + HOST_GID: {self.HOST_GID} + TZ: {self.TZ} + ENABLE_CAST: {self.ENABLE_CAST} + TA_PORT: {self.TA_PORT} + TA_UWSGI_PORT: {self.TA_UWSGI_PORT} + TA_USERNAME: {self.TA_USERNAME} + TA_PASSWORD: *****""" + ) + + def print_paths(self): + """debug paths set""" + print( + f""" + MEDIA_DIR: {self.MEDIA_DIR} + APP_DIR: {self.APP_DIR} + CACHE_DIR: {self.CACHE_DIR}""" + ) + + def print_redis_conf(self): + """debug redis conf paths""" + print( + f""" + REDIS_HOST: {self.REDIS_HOST} + REDIS_PORT: {self.REDIS_PORT} + REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}""" + ) + + def print_es_paths(self): + """debug es conf""" + print( + f""" + ES_URL: {self.ES_URL} + ES_PASS: ***** + ES_USER: {self.ES_USER} + ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}""" + ) + + def print_all(self): + """print all""" + self.print_generic() + self.print_paths() + self.print_redis_conf() + self.print_es_paths() From 2826ca4a4300e20b18c8c608f5b7a3ee7d82e1fe Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 28 Oct 2023 15:25:57 +0700 Subject: [PATCH 10/13] move ES_SNAPSHOT_DIR to EnvironmentSettings --- tubearchivist/config/management/commands/ta_connection.py | 8 +++----- tubearchivist/home/src/es/snapshot.py | 5 +---- tubearchivist/home/src/ta/settings.py | 6 ++++++ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_connection.py b/tubearchivist/config/management/commands/ta_connection.py index 4e0f59aa..94b5e0dc 100644 --- a/tubearchivist/config/management/commands/ta_connection.py +++ b/tubearchivist/config/management/commands/ta_connection.py @@ -3,12 +3,12 @@ Functionality: - check that all connections are working """ -from os import environ from time import sleep import requests from django.core.management.base import BaseCommand, CommandError from home.src.es.connect import ElasticWrap +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist TOPIC = """ @@ -156,10 +156,8 @@ class Command(BaseCommand): " 🗙 path.repo env var not found. " + "set the following env var to the ES container:\n" + " path.repo=" - + environ.get( - "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" - ), + + EnvironmentSettings.ES_SNAPSHOT_DIR ) - self.stdout.write(self.style.ERROR(f"{message}")) + self.stdout.write(self.style.ERROR(message)) sleep(60) raise CommandError(message) diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index d77f0761..0cff51e6 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -4,7 +4,6 @@ functionality: """ from datetime import datetime -from os import environ from time import sleep from zoneinfo import ZoneInfo @@ -20,9 +19,7 @@ class ElasticSnapshot: REPO_SETTINGS = { "compress": "true", "chunk_size": "1g", - "location": environ.get( - "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" - ), + "location": EnvironmentSettings.ES_SNAPSHOT_DIR, } POLICY = "ta_daily" diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py index db609c76..16f36994 100644 --- a/tubearchivist/home/src/ta/settings.py +++ b/tubearchivist/home/src/ta/settings.py @@ -37,6 +37,11 @@ class EnvironmentSettings: ES_URL: str = str(environ.get("ES_URL")) ES_PASS: str = str(environ.get("ELASTIC_PASSWORD")) ES_USER: str = str(environ.get("ELASTIC_USER", "elastic")) + ES_SNAPSHOT_DIR: str = str( + environ.get( + "ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot" + ) + ) ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL")) def print_generic(self): @@ -78,6 +83,7 @@ class EnvironmentSettings: ES_URL: {self.ES_URL} ES_PASS: ***** ES_USER: {self.ES_USER} + ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR} ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}""" ) From 8247314d01da8ef3e10071aa47dde35c3d9730a8 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 31 Oct 2023 15:50:33 +0700 Subject: [PATCH 11/13] refactor admin permisson classes --- tubearchivist/api/views.py | 62 ++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 54c88fba..bb72322b 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -2,8 +2,6 @@ from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress from api.src.search_processor import SearchProcess -from django.contrib.auth.decorators import user_passes_test -from django.utils.decorators import method_decorator from home.src.download.queue import PendingInteract from home.src.download.subscriptions import ( ChannelSubscription, @@ -31,27 +29,44 @@ from home.tasks import ( extrac_dl, subscribe_to, ) +from rest_framework import permissions from rest_framework.authentication import ( SessionAuthentication, TokenAuthentication, ) from rest_framework.authtoken.models import Token from rest_framework.authtoken.views import ObtainAuthToken -from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView def check_admin(user): - """if user has access to restricted views""" + """check for admin permission for restricted views""" return user.is_staff or user.groups.filter(name="admin").exists() +class AdminOnly(permissions.BasePermission): + """allow only admin""" + + def has_permission(self, request, view): + return check_admin(request.user) + + +class AdminWriteOnly(permissions.BasePermission): + """allow only admin writes""" + + def has_permission(self, request, view): + if request.method in permissions.SAFE_METHODS: + return permissions.IsAuthenticated().has_permission(request, view) + + return check_admin(request.user) + + class ApiBaseView(APIView): """base view to inherit from""" authentication_classes = [SessionAuthentication, TokenAuthentication] - permission_classes = [IsAuthenticated] + permission_classes = [permissions.IsAuthenticated] search_base = "" data = "" @@ -116,6 +131,7 @@ class VideoApiView(ApiBaseView): """ search_base = "ta_video/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, video_id): # pylint: disable=unused-argument @@ -123,7 +139,6 @@ class VideoApiView(ApiBaseView): self.get_document(video_id) return Response(self.response, status=self.status_code) - @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, video_id): # pylint: disable=unused-argument """delete single video""" @@ -290,6 +305,7 @@ class ChannelApiView(ApiBaseView): """ search_base = "ta_channel/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, channel_id): # pylint: disable=unused-argument @@ -297,7 +313,6 @@ class ChannelApiView(ApiBaseView): self.get_document(channel_id) return Response(self.response, status=self.status_code) - @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, channel_id): # pylint: disable=unused-argument """delete channel""" @@ -321,6 +336,7 @@ class ChannelApiListView(ApiBaseView): search_base = "ta_channel/_search/" valid_filter = ["subscribed"] + permission_classes = [AdminWriteOnly] def get(self, request): """get request""" @@ -343,7 +359,6 @@ class ChannelApiListView(ApiBaseView): return Response(self.response) - @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request): """subscribe/unsubscribe to list of channels""" data = request.data @@ -435,6 +450,7 @@ class PlaylistApiListView(ApiBaseView): """ search_base = "ta_playlist/_search/" + permission_classes = [AdminWriteOnly] def get(self, request): """handle get request""" @@ -444,7 +460,6 @@ class PlaylistApiListView(ApiBaseView): self.get_document_list(request) return Response(self.response) - @method_decorator(user_passes_test(check_admin), name="dispatch") def post(self, request): """subscribe/unsubscribe to list of playlists""" data = request.data @@ -484,6 +499,7 @@ class PlaylistApiView(ApiBaseView): """ search_base = "ta_playlist/_doc/" + permission_classes = [AdminWriteOnly] def get(self, request, playlist_id): # pylint: disable=unused-argument @@ -491,7 +507,6 @@ class PlaylistApiView(ApiBaseView): self.get_document(playlist_id) return Response(self.response, status=self.status_code) - @method_decorator(user_passes_test(check_admin), name="dispatch") def delete(self, request, playlist_id): """delete playlist""" print(f"{playlist_id}: delete playlist") @@ -522,7 +537,6 @@ class PlaylistApiVideoView(ApiBaseView): return Response(self.response, status=self.status_code) -@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadApiView(ApiBaseView): """resolves to /api/download// GET: returns metadata dict of an item in the download queue @@ -532,6 +546,7 @@ class DownloadApiView(ApiBaseView): search_base = "ta_download/_doc/" valid_status = ["pending", "ignore", "priority"] + permission_classes = [AdminOnly] def get(self, request, video_id): # pylint: disable=unused-argument @@ -569,7 +584,6 @@ class DownloadApiView(ApiBaseView): return Response({"success": True}) -@method_decorator(user_passes_test(check_admin), name="dispatch") class DownloadApiListView(ApiBaseView): """resolves to /api/download/ GET: returns latest videos in the download queue @@ -579,6 +593,7 @@ class DownloadApiListView(ApiBaseView): search_base = "ta_download/_search/" valid_filter = ["pending", "ignore"] + permission_classes = [AdminOnly] def get(self, request): """get request""" @@ -681,13 +696,14 @@ class LoginApiView(ObtainAuthToken): return Response({"token": token.key, "user_id": user.pk}) -@method_decorator(user_passes_test(check_admin), name="dispatch") class SnapshotApiListView(ApiBaseView): """resolves to /api/snapshot/ GET: returns snapshot config plus list of existing snapshots POST: take snapshot now """ + permission_classes = [AdminOnly] + @staticmethod def get(request): """handle get request""" @@ -705,7 +721,6 @@ class SnapshotApiListView(ApiBaseView): return Response(response) -@method_decorator(user_passes_test(check_admin), name="dispatch") class SnapshotApiView(ApiBaseView): """resolves to /api/snapshot// GET: return a single snapshot @@ -713,6 +728,8 @@ class SnapshotApiView(ApiBaseView): DELETE: delete a snapshot """ + permission_classes = [AdminOnly] + @staticmethod def get(request, snapshot_id): """handle get request""" @@ -747,12 +764,13 @@ class SnapshotApiView(ApiBaseView): return Response(response) -@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskListView(ApiBaseView): """resolves to /api/task-name/ GET: return a list of all stored task results """ + permission_classes = [AdminOnly] + def get(self, request): """handle get request""" # pylint: disable=unused-argument @@ -761,13 +779,14 @@ class TaskListView(ApiBaseView): return Response(all_results) -@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskNameListView(ApiBaseView): """resolves to /api/task-name// GET: return a list of stored results of task POST: start new background process """ + permission_classes = [AdminOnly] + def get(self, request, task_name): """handle get request""" # pylint: disable=unused-argument @@ -800,13 +819,13 @@ class TaskNameListView(ApiBaseView): return Response({"message": message}) -@method_decorator(user_passes_test(check_admin), name="dispatch") class TaskIDView(ApiBaseView): """resolves to /api/task-id// GET: return details of task id """ valid_commands = ["stop", "kill"] + permission_classes = [AdminOnly] def get(self, request, task_id): """handle get request""" @@ -852,13 +871,14 @@ class TaskIDView(ApiBaseView): return f"message:{task_conf.get('group')}:{task_id.split('-')[0]}" -@method_decorator(user_passes_test(check_admin), name="dispatch") class RefreshView(ApiBaseView): """resolves to /api/refresh/ GET: get refresh progress POST: start a manual refresh task """ + permission_classes = [AdminOnly] + def get(self, request): """handle get request""" request_type = request.GET.get("type") @@ -885,7 +905,6 @@ class RefreshView(ApiBaseView): return Response(data) -@method_decorator(user_passes_test(check_admin), name="dispatch") class CookieView(ApiBaseView): """resolves to /api/cookie/ GET: check if cookie is enabled @@ -893,6 +912,8 @@ class CookieView(ApiBaseView): PUT: import cookie """ + permission_classes = [AdminOnly] + @staticmethod def get(request): """handle get request""" @@ -975,12 +996,13 @@ class SearchView(ApiBaseView): return Response(search_results) -@method_decorator(user_passes_test(check_admin), name="dispatch") class TokenView(ApiBaseView): """resolves to /api/token/ DELETE: revoke the token """ + permission_classes = [AdminOnly] + @staticmethod def delete(request): print("revoke API token") From aa475c58aac1795cb975ce853bf54692610018e5 Mon Sep 17 00:00:00 2001 From: Merlin <4706504+MerlinScheurer@users.noreply.github.com> Date: Wed, 1 Nov 2023 02:40:41 +0100 Subject: [PATCH 12/13] Refac settings dashboard (#577) * Add padding to duration str text * Add singular and plural to video in dailyStat * Add code spacing for readability * Refac Main overview in dashboard to be spaced evenly and use tables * Refac simplify number padding * Refac skip adding spacing rows on mobile * Refac reorder watch progress to be in order of interest * Fix that ther can be 0 Videos added a day * Refac capitalize content keys --- tubearchivist/api/src/aggs.py | 2 +- tubearchivist/home/src/ta/helper.py | 2 +- tubearchivist/static/stats.js | 193 ++++++++++++++++++++++------ 3 files changed, 158 insertions(+), 39 deletions(-) diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py index 1e2f54c6..ac6c2e69 100644 --- a/tubearchivist/api/src/aggs.py +++ b/tubearchivist/api/src/aggs.py @@ -117,7 +117,7 @@ class WatchProgress(AggBase): all_duration = int(aggregations["total_duration"].get("value")) response.update( { - "all": { + "total": { "duration": all_duration, "duration_str": get_duration_str(all_duration), "items": aggregations["total_vids"].get("value"), diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 4143ed6e..8bce8098 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -178,7 +178,7 @@ def get_duration_str(seconds: int) -> str: for unit_label, unit_seconds in units: if seconds >= unit_seconds: unit_count, seconds = divmod(seconds, unit_seconds) - duration_parts.append(f"{unit_count}{unit_label}") + duration_parts.append(f"{unit_count:02}{unit_label}") return " ".join(duration_parts) diff --git a/tubearchivist/static/stats.js b/tubearchivist/static/stats.js index 23732667..3336753d 100644 --- a/tubearchivist/static/stats.js +++ b/tubearchivist/static/stats.js @@ -8,13 +8,18 @@ function primaryStats() { let apiEndpoint = '/api/stats/primary/'; let responseData = apiRequest(apiEndpoint, 'GET'); let primaryBox = document.getElementById('primaryBox'); + clearLoading(primaryBox); + let videoTile = buildVideoTile(responseData); primaryBox.appendChild(videoTile); + let channelTile = buildChannelTile(responseData); primaryBox.appendChild(channelTile); + let playlistTile = buildPlaylistTile(responseData); primaryBox.appendChild(playlistTile); + let downloadTile = buildDownloadTile(responseData); primaryBox.appendChild(downloadTile); } @@ -26,51 +31,133 @@ function clearLoading(dashBox) { function buildTile(titleText) { let tile = document.createElement('div'); tile.classList.add('info-box-item'); + let title = document.createElement('h3'); + title.innerText = titleText; tile.appendChild(title); + return tile; } +function buildTileContenTable(content, rowsWanted) { + let contentEntries = Object.entries(content); + + const nbsp = '\u00A0'; // No-Break Space https://www.compart.com/en/unicode/U+00A0 + + // Do not add spacing rows when on mobile device + const isMobile = window.matchMedia('(max-width: 600px)'); + if (!isMobile.matches) { + if (contentEntries.length < rowsWanted) { + const rowsToAdd = rowsWanted - contentEntries.length; + + for (let i = 0; i < rowsToAdd; i++) { + contentEntries.push([nbsp, nbsp]); + } + } + } + + const table = document.createElement('table'); + table.classList.add('agg-channel-table'); + const tableBody = document.createElement('tbody'); + + for (const [key, value] of contentEntries) { + const row = document.createElement('tr'); + + const leftCell = document.createElement('td'); + leftCell.classList.add('agg-channel-name'); + + // Do not add ":" when its a spacing entry + const keyText = key === nbsp ? key : `${key}: `; + const leftText = document.createTextNode(keyText); + leftCell.appendChild(leftText); + + const rightCell = document.createElement('td'); + rightCell.classList.add('agg-channel-right-align'); + + const rightText = document.createTextNode(value); + rightCell.appendChild(rightText); + + row.appendChild(leftCell); + row.appendChild(rightCell); + + tableBody.appendChild(row); + } + + table.appendChild(tableBody); + + return table; +} + function buildVideoTile(responseData) { - let tile = buildTile(`Total Videos: ${responseData.videos.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = ` - videos: ${responseData.videos.videos || 0}
- shorts: ${responseData.videos.shorts || 0}
- streams: ${responseData.videos.streams || 0}
- `; - tile.appendChild(message); + let tile = buildTile(`Video types: `); + + const total = responseData.videos.total || 0; + const videos = responseData.videos.videos || 0; + const shorts = responseData.videos.shorts || 0; + const streams = responseData.videos.streams || 0; + + const content = { + Videos: `${videos}/${total}`, + Shorts: `${shorts}/${total}`, + Streams: `${streams}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildChannelTile(responseData) { - let tile = buildTile(`Total Channels: ${responseData.channels.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = `subscribed: ${responseData.channels.sub_true || 0}`; - tile.appendChild(message); + let tile = buildTile(`Channels: `); + + const total = responseData.channels.total || 0; + const subscribed = responseData.channels.sub_true || 0; + + const content = { + Subscribed: `${subscribed}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildPlaylistTile(responseData) { - let tile = buildTile(`Total Playlists: ${responseData.playlists.total || 0}`); - let message = document.createElement('p'); - message.innerHTML = `subscribed: ${responseData.playlists.sub_true || 0}`; - tile.appendChild(message); + let tile = buildTile(`Playlists:`); + + const total = responseData.playlists.total || 0; + const subscribed = responseData.playlists.sub_true || 0; + + const content = { + Subscribed: `${subscribed}/${total}`, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } function buildDownloadTile(responseData) { let tile = buildTile('Downloads'); - let message = document.createElement('p'); - message.innerHTML = ` - pending: ${responseData.downloads.pending || 0}
- ignored: ${responseData.downloads.ignore || 0}
- `; - tile.appendChild(message); + + const pending = responseData.downloads.pending || 0; + const ignored = responseData.downloads.ignore || 0; + + const content = { + Pending: pending, + Ignored: ignored, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); return tile; } @@ -81,24 +168,48 @@ function watchStats() { let watchBox = document.getElementById('watchBox'); clearLoading(watchBox); - for (const property in responseData) { - let tile = buildWatchTile(property, responseData[property]); - watchBox.appendChild(tile); - } + const { total, watched, unwatched } = responseData; + + let firstCard = buildWatchTile('total', total); + watchBox.appendChild(firstCard); + + let secondCard = buildWatchTile('watched', watched); + watchBox.appendChild(secondCard); + + let thirdCard = buildWatchTile('unwatched', unwatched); + watchBox.appendChild(thirdCard); +} + +function capitalizeFirstLetter(string) { + // source: https://stackoverflow.com/a/1026087 + return string.charAt(0).toUpperCase() + string.slice(1); } function buildWatchTile(title, watchDetail) { - let tile = buildTile(`Total ${title}`); - let message = document.createElement('p'); - message.innerHTML = ` - ${watchDetail.items} Videos
- ${watchDetail.duration} Seconds
- ${watchDetail.duration_str} Playback - `; - if (watchDetail.progress) { - message.innerHTML += `
${Number(watchDetail.progress * 100).toFixed(2)}%`; + const items = watchDetail.items || 0; + const duration = watchDetail.duration || 0; + const duration_str = watchDetail.duration_str || 0; + const hasProgess = !!watchDetail.progress; + const progress = Number(watchDetail.progress * 100).toFixed(2); + + let titleCapizalized = capitalizeFirstLetter(title); + + if (hasProgess) { + titleCapizalized = `${progress}% ` + titleCapizalized; } - tile.appendChild(message); + + let tile = buildTile(titleCapizalized); + + const content = { + Videos: items, + Seconds: duration, + Playback: duration_str, + }; + + const table = buildTileContenTable(content, 3); + + tile.appendChild(table); + return tile; } @@ -123,7 +234,15 @@ function downloadHist() { function buildDailyStat(dailyStat) { let tile = buildTile(dailyStat.date); let message = document.createElement('p'); - message.innerText = `new videos: ${dailyStat.count}`; + const isExactlyOne = dailyStat.count === 1; + + let text = 'Videos'; + if (isExactlyOne) { + text = 'Video'; + } + + message.innerText = `+${dailyStat.count} ${text}`; + tile.appendChild(message); return tile; } From 58ea256b4462163232cb84a162d73c1f01534ccd Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 09:19:18 +0700 Subject: [PATCH 13/13] add unstable tag --- tubearchivist/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index f4106551..765a6f2b 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -269,4 +269,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [ # TA application settings TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist" -TA_VERSION = "v0.4.2" +TA_VERSION = "v0.4.3-unstable"