tubearchivist/tubearchivist
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist.git synced 2025-03-18 11:50:12 +00:00
Code Issues Projects Releases Wiki Activity

Data Serializers, #build

Browse Source 
Changed:
- Serialize all data
- API simplify item return format
- API document with swagger docs from serializers
This commit is contained in:
Simon 2025-02-18 23:13:23 +07:00
commit 464619cc00
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
89 changed files with 3928 additions and 1416 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -36,7 +36,7 @@ FROM python:3.11.8-slim-bookworm AS tubearchivist
ARG INSTALL_DEBUG
ENV PYTHONUNBUFFERED 1
ENV PYTHONUNBUFFERED=1
# copy build requirements
COPY --from=builder /root/.local /root/.local

132
backend/appsettings/serializers.py Normal file
View File

@ -0,0 +1,132 @@
"""appsettings erializers"""
# pylint: disable=abstract-method
from common.serializers import ValidateUnknownFieldsMixin
from rest_framework import serializers
class BackupFileSerializer(serializers.Serializer):
"""serialize backup file"""
filename = serializers.CharField()
file_path = serializers.CharField()
file_size = serializers.IntegerField()
timestamp = serializers.CharField()
reason = serializers.CharField()
class AppConfigSubSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config subscriptions"""
channel_size = serializers.IntegerField(required=False)
live_channel_size = serializers.IntegerField(required=False)
shorts_channel_size = serializers.IntegerField(required=False)
auto_start = serializers.BooleanField(required=False)
class AppConfigDownloadsSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config downloads config"""
limit_speed = serializers.IntegerField(allow_null=True)
sleep_interval = serializers.IntegerField(allow_null=True)
autodelete_days = serializers.IntegerField(allow_null=True)
format = serializers.CharField(allow_null=True)
format_sort = serializers.CharField(allow_null=True)
add_metadata = serializers.BooleanField()
add_thumbnail = serializers.BooleanField()
subtitle = serializers.CharField(allow_null=True)
subtitle_source = serializers.ChoiceField(
choices=["auto", "user"], allow_null=True
)
subtitle_index = serializers.BooleanField()
comment_max = serializers.CharField(allow_null=True)
comment_sort = serializers.ChoiceField(
choices=["top", "new"], allow_null=True
)
cookie_import = serializers.BooleanField()
potoken = serializers.BooleanField()
throttledratelimit = serializers.IntegerField(allow_null=True)
extractor_lang = serializers.CharField(allow_null=True)
integrate_ryd = serializers.BooleanField()
integrate_sponsorblock = serializers.BooleanField()
class AppConfigAppSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize app config"""
enable_snapshot = serializers.BooleanField()
class AppConfigSerializer(ValidateUnknownFieldsMixin, serializers.Serializer):
"""serialize appconfig"""
subscriptions = AppConfigSubSerializer(required=False)
downloads = AppConfigDownloadsSerializer(required=False)
application = AppConfigAppSerializer(required=False)
class CookieValidationSerializer(serializers.Serializer):
"""serialize cookie validation response"""
cookie_enabled = serializers.BooleanField()
status = serializers.BooleanField(required=False)
validated = serializers.IntegerField(required=False)
validated_str = serializers.CharField(required=False)
class CookieUpdateSerializer(serializers.Serializer):
"""serialize cookie to update"""
cookie = serializers.CharField()
class PoTokenSerializer(serializers.Serializer):
"""serialize PO token"""
potoken = serializers.CharField()
class SnapshotItemSerializer(serializers.Serializer):
"""serialize snapshot response"""
id = serializers.CharField()
state = serializers.CharField()
es_version = serializers.CharField()
start_date = serializers.CharField()
end_date = serializers.CharField()
end_stamp = serializers.IntegerField()
duration_s = serializers.IntegerField()
class SnapshotListSerializer(serializers.Serializer):
"""serialize snapshot list response"""
next_exec = serializers.IntegerField()
next_exec_str = serializers.CharField()
expire_after = serializers.CharField()
snapshots = SnapshotItemSerializer(many=True)
class SnapshotCreateResponseSerializer(serializers.Serializer):
"""serialize new snapshot creating response"""
snapshot_name = serializers.CharField()
class SnapshotRestoreResponseSerializer(serializers.Serializer):
"""serialize snapshot restore response"""
accepted = serializers.BooleanField()
class TokenResponseSerializer(serializers.Serializer):
"""serialize token response"""
token = serializers.CharField()

37
backend/appsettings/src/config.py
View File

@ -109,29 +109,24 @@ class AppConfig:
def update_config(self, data: dict) -> AppConfigType:
"""update single config value"""
new_config = self.config.copy()
for key, value in data.items():
key_map = key.split(".")
self._validate_key(key_map)
self.config[key_map[0]][key_map[1]] = value
if (
isinstance(value, dict)
and key in new_config
and isinstance(new_config[key], dict)
):
new_config[key].update(value)
else:
new_config[key] = value
response, status_code = ElasticWrap(self.ES_PATH).post(self.config)
response, status_code = ElasticWrap(self.ES_PATH).post(new_config)
if not status_code == 200:
print(response)
return self.config
self.config = new_config
def _update_config_dict(self, to_update) -> None:
"""none validated partial update for defaults sync"""
data = {"doc": to_update}
response, status_code = ElasticWrap(self.ES_UPDATE_PATH).post(data)
if not status_code == 200:
print(f"update failed: {response}, {status_code}")
def _validate_key(self, key_map: list[str]) -> None:
"""raise valueerror on invalid key"""
exists = key_map[1] in self.CONFIG_DEFAULTS.get(key_map[0], {}) # type: ignore # noqa: E501
if exists is None:
raise ValueError(f"trying to access invalid config key: {key_map}")
return new_config
def post_process_updated(self, data: dict) -> None:
"""apply hooks for some config keys"""
@ -163,7 +158,7 @@ class AppConfig:
for key, value in self.CONFIG_DEFAULTS.items():
if key not in self.config:
# complete new key
self._update_config_dict({key: value})
self.update_config({key: value})
updated.append(str({key: value}))
continue
@ -171,7 +166,7 @@ class AppConfig:
if sub_key not in self.config[key]:
# new partial key
to_update = {key: {sub_key: sub_value}}
self._update_config_dict(to_update)
self.update_config(to_update)
updated.append(str(to_update))
return updated
@ -247,7 +242,7 @@ class ReleaseVersion:
return False
def get_update(self) -> dict:
def get_update(self) -> dict | None:
"""return new version dict if available"""
message = RedisArchivist().get_message_dict(self.NEW_KEY)
return message
return message or None

35
backend/appsettings/src/filesystem.py
View File

@ -6,7 +6,7 @@ Functionality:
import os
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap, IndexPaginate
from common.src.es_connect import IndexPaginate
from common.src.helper import ignore_filelist
from video.src.comments import CommentList
from video.src.index import YoutubeVideo, index_new_video
@ -56,7 +56,6 @@ class Scanner:
"""apply all changes"""
self.delete()
self.index()
self.url_fix()
def delete(self) -> None:
"""delete videos from index"""
@ -92,35 +91,3 @@ class Scanner:
comment_list = CommentList(task=self.task)
comment_list.add(video_ids=list(self.to_index))
comment_list.index()
def url_fix(self) -> None:
"""
update path v0.3.6 to v0.3.7
fix url not matching channel-videoid pattern
"""
bool_must = (
"doc['media_url'].value == "
+ "(doc['channel.channel_id'].value + '/' + "
+ "doc['youtube_id'].value) + '.mp4'"
)
to_update = (
"ctx._source['media_url'] = "
+ "ctx._source.channel['channel_id'] + '/' + "
+ "ctx._source['youtube_id'] + '.mp4'"
)
data = {
"query": {
"bool": {
"must_not": [{"script": {"script": {"source": bool_must}}}]
}
},
"script": {"source": to_update},
}
response, _ = ElasticWrap("ta_video/_update_by_query").post(data=data)
updated = response.get("updates")
if updated:
print(f"updated {updated} bad media_url")
if self.task:
self.task.send_progress(
[f"Updated {updated} wrong media urls."]
)

1
backend/appsettings/src/reindex.py
View File

@ -333,6 +333,7 @@ class Reindex(ReindexBase):
# add back
video.json_data["player"] = es_meta.get("player")
video.json_data["date_downloaded"] = es_meta.get("date_downloaded")
video.json_data["video_type"] = es_meta.get("video_type")
video.json_data["channel"] = es_meta.get("channel")
if es_meta.get("playlist"):
video.json_data["playlist"] = es_meta.get("playlist")

463
backend/appsettings/views.py
View File

@ -1,116 +1,35 @@
"""all app settings API views"""
from appsettings.serializers import (
AppConfigSerializer,
BackupFileSerializer,
CookieUpdateSerializer,
CookieValidationSerializer,
PoTokenSerializer,
SnapshotCreateResponseSerializer,
SnapshotItemSerializer,
SnapshotListSerializer,
SnapshotRestoreResponseSerializer,
TokenResponseSerializer,
)
from appsettings.src.backup import ElasticBackup
from appsettings.src.config import AppConfig
from appsettings.src.snapshot import ElasticSnapshot
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
)
from common.src.ta_redis import RedisArchivist
from common.views_base import AdminOnly, ApiBaseView
from django.conf import settings
from download.src.yt_dlp_base import CookieHandler, POTokenHandler
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.authtoken.models import Token
from rest_framework.response import Response
from task.src.task_manager import TaskCommand
from task.tasks import run_restore_backup
class AppConfigApiView(ApiBaseView):
"""resolves to /api/appsettings/config/
GET: return app settings
POST: update app settings
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""get config"""
response = AppConfig().config
return Response(response)
@staticmethod
def post(request):
"""
update config values
data object where key is flatted CONFIG_DEFAULTS separated by '.', e.g.
{"subscriptions.channel_size": 5, "subscriptions.live_channel_size": 5}
"""
data = request.data
try:
config = AppConfig().update_config(data)
except ValueError as err:
return Response({"error": str(err)}, status=400)
return Response(config)
class SnapshotApiListView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/
GET: returns snapshot config plus list of existing snapshots
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
# pylint: disable=unused-argument
snapshots = ElasticSnapshot().get_snapshot_stats()
return Response(snapshots)
@staticmethod
def post(request):
"""take snapshot now with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().take_snapshot_now()
return Response(response)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot
POST: restore snapshot
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request, snapshot_id):
"""handle get request"""
# pylint: disable=unused-argument
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
if not snapshot:
return Response({"message": "snapshot not found"}, status=404)
return Response(snapshot)
@staticmethod
def post(request, snapshot_id):
"""restore snapshot with post request"""
# pylint: disable=unused-argument
response = ElasticSnapshot().restore_all(snapshot_id)
if not response:
message = {"message": "failed to restore snapshot"}
return Response(message, status=400)
return Response(response)
@staticmethod
def delete(request, snapshot_id):
"""delete snapshot from index"""
# pylint: disable=unused-argument
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
if not response:
message = {"message": "failed to delete snapshot"}
return Response(message, status=400)
return Response(response)
class BackupApiListView(ApiBaseView):
"""resolves to /api/appsettings/backup/
GET: returns list of available zip backups
@ -121,22 +40,34 @@ class BackupApiListView(ApiBaseView):
task_name = "run_backup"
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(BackupFileSerializer(many=True)),
},
)
def get(request):
"""handle get request"""
"""get list of available backup files"""
# pylint: disable=unused-argument
backup_files = ElasticBackup().get_all_backup_files()
return Response(backup_files)
serializer = BackupFileSerializer(backup_files, many=True)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
},
)
def post(self, request):
"""handle post request"""
"""start new backup file task"""
# pylint: disable=unused-argument
response = TaskCommand().start(self.task_name)
message = {
"message": "backup task started",
"task_id": response["task_id"],
}
serializer = AsyncTaskResponseSerializer(message)
return Response(message)
return Response(serializer.data)
class BackupApiView(ApiBaseView):
@ -150,19 +81,42 @@ class BackupApiView(ApiBaseView):
task_name = "restore_backup"
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(BackupFileSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def get(request, filename):
"""get single backup"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().build_backup_file_data(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
return Response(backup_file)
serializer = BackupFileSerializer(backup_file)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def post(self, request, filename):
"""restore backup file"""
"""start new task to restore backup file"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().build_backup_file_data(filename)
if not backup_file:
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
task = run_restore_backup.delay(filename)
message = {
"message": "backup restore task started",
@ -172,17 +126,64 @@ class BackupApiView(ApiBaseView):
return Response(message)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="file deleted"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="file not found"
),
}
)
def delete(request, filename):
"""delete backup file"""
# pylint: disable=unused-argument
backup_file = ElasticBackup().delete_file(filename)
if not backup_file:
message = {"message": "file not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "file not found"})
return Response(error.data, status=404)
message = {"message": f"file {filename} deleted"}
return Response(message)
return Response(status=204)
class AppConfigApiView(ApiBaseView):
"""resolves to /api/appsettings/config/
GET: return app settings
POST: update app settings
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(AppConfigSerializer()),
}
)
def get(request):
"""get app config"""
response = AppConfig().config
serializer = AppConfigSerializer(response)
return Response(serializer.data)
@staticmethod
@extend_schema(
request=AppConfigSerializer(),
responses={
200: OpenApiResponse(AppConfigSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(request):
"""update config values, allows partial update"""
serializer = AppConfigSerializer(data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
updated_config = AppConfig().update_config(validated_data)
updated_serializer = AppConfigSerializer(updated_config)
return Response(updated_serializer.data)
class CookieView(ApiBaseView):
@ -195,54 +196,88 @@ class CookieView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(CookieValidationSerializer()),
}
)
def get(self, request):
"""handle get request"""
"""get cookie validation status"""
# pylint: disable=unused-argument
validation = self._get_cookie_validation()
serializer = CookieValidationSerializer(validation)
return Response(validation)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(CookieValidationSerializer()),
}
)
def post(self, request):
"""handle cookie validation request"""
"""validate cookie"""
# pylint: disable=unused-argument
config = AppConfig().config
_ = CookieHandler(config).validate()
validation = self._get_cookie_validation()
serializer = CookieValidationSerializer(validation)
return Response(validation)
return Response(serializer.data)
@extend_schema(
request=CookieUpdateSerializer(),
responses={
200: OpenApiResponse(CookieValidationSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def put(self, request):
"""handle put request"""
# pylint: disable=unused-argument
config = AppConfig().config
cookie = request.data.get("cookie")
serializer = CookieUpdateSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
cookie = validated_data.get("cookie")
if not cookie:
message = "missing cookie key in request data"
print(message)
return Response({"message": message}, status=400)
error = ErrorResponseSerializer({"error": message})
return Response(error.data, status=400)
if settings.DEBUG:
print(f"[cookie] preview:\n\n{cookie[:300]}")
config = AppConfig().config
handler = CookieHandler(config)
handler.set_cookie(cookie)
validated = handler.validate()
if not validated:
message = "[cookie]: import failed, not valid"
print(message)
error = ErrorResponseSerializer({"error": message})
handler.revoke()
print("[cookie]: import failed, not valid")
status = 400
else:
status = 200
return Response(error.data, status=400)
validation = self._get_cookie_validation()
return Response(validation, status=status)
serializer = CookieValidationSerializer(validation)
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="Cookie revoked"),
},
)
def delete(self, request):
"""delete the cookie"""
config = AppConfig().config
handler = CookieHandler(config)
handler.revoke()
return Response({"cookie_enabled": False})
return Response(status=204)
@staticmethod
def _get_cookie_validation():
@ -260,47 +295,199 @@ class POTokenView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(PoTokenSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="PO token not found"
),
}
)
def get(self, request):
"""get token"""
"""get PO token"""
config = AppConfig().config
potoken = POTokenHandler(config).get()
return Response({"potoken": potoken})
if not potoken:
error = ErrorResponseSerializer({"error": "PO token not found"})
return Response(error.data, status=404)
serializer = PoTokenSerializer(data={"potoken": potoken})
serializer.is_valid(raise_exception=True)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(PoTokenSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
}
)
def post(self, request):
"""post token"""
"""Update PO token"""
serializer = PoTokenSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
if not validated_data:
error = ErrorResponseSerializer(
{"error": "missing PO token key in request data"}
)
return Response(error.data, status=400)
config = AppConfig().config
new_token = request.data.get("potoken")
if not new_token:
message = "missing potoken key in request data"
print(message)
return Response({"message": message}, status=400)
new_token = validated_data["potoken"]
POTokenHandler(config).set_token(new_token)
return Response({"potoken": new_token})
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="PO token revoked"),
},
)
def delete(self, request):
"""delete token"""
"""delete PO token"""
config = AppConfig().config
POTokenHandler(config).revoke_token()
return Response({"potoken": None})
return Response(status=204)
class SnapshotApiListView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/
GET: returns snapshot config plus list of existing snapshots
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotListSerializer()),
}
)
def get(request):
"""get available snapshots with metadata"""
# pylint: disable=unused-argument
snapshots = ElasticSnapshot().get_snapshot_stats()
serializer = SnapshotListSerializer(snapshots)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotCreateResponseSerializer()),
}
)
def post(request):
"""take snapshot now"""
# pylint: disable=unused-argument
response = ElasticSnapshot().take_snapshot_now()
serializer = SnapshotCreateResponseSerializer(response)
return Response(serializer.data)
class SnapshotApiView(ApiBaseView):
"""resolves to /api/appsettings/snapshot/<snapshot-id>/
GET: return a single snapshot
POST: restore snapshot
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotItemSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="snapshot not found"
),
}
)
def get(request, snapshot_id):
"""handle get request"""
# pylint: disable=unused-argument
snapshot = ElasticSnapshot().get_single_snapshot(snapshot_id)
if not snapshot:
error = ErrorResponseSerializer({"error": "snapshot not found"})
return Response(error.data, status=404)
serializer = SnapshotItemSerializer(snapshot)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(SnapshotRestoreResponseSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
}
)
def post(request, snapshot_id):
"""restore snapshot"""
# pylint: disable=unused-argument
response = ElasticSnapshot().restore_all(snapshot_id)
if not response:
error = ErrorResponseSerializer(
{"error": "failed to restore snapshot"}
)
return Response(error.data, status=400)
serializer = SnapshotRestoreResponseSerializer(response)
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="delete snapshot from index"),
}
)
def delete(request, snapshot_id):
"""delete snapshot from index"""
# pylint: disable=unused-argument
response = ElasticSnapshot().delete_single_snapshot(snapshot_id)
if not response:
error = ErrorResponseSerializer(
{"error": "failed to delete snapshot"}
)
return Response(error.data, status=400)
return Response(status=204)
class TokenView(ApiBaseView):
"""resolves to /api/appsettings/token/
GET: get API token
DELETE: revoke the token
"""
permission_classes = [AdminOnly]
@staticmethod
@extend_schema(
responses={
200: OpenApiResponse(TokenResponseSerializer()),
}
)
def get(request):
"""get token"""
"""get your API token"""
token, _ = Token.objects.get_or_create(user=request.user)
return Response({"token": token.key})
serializer = TokenResponseSerializer({"token": token.key})
return Response(serializer.data)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="delete token"),
}
)
def delete(request):
"""delete the token, new will get created automatically"""
"""delete your API token, new will get created on next get"""
print("revoke API token")
request.user.auth_token.delete()
return Response({"success": True})
return Response(status=204)

99
backend/channel/serializers.py Normal file
View File

@ -0,0 +1,99 @@
"""channel serializers"""
# pylint: disable=abstract-method
from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
from rest_framework import serializers
class ChannelOverwriteSerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize channel overwrites"""
download_format = serializers.CharField(required=False, allow_null=True)
autodelete_days = serializers.IntegerField(required=False, allow_null=True)
index_playlists = serializers.BooleanField(required=False, allow_null=True)
integrate_sponsorblock = serializers.BooleanField(
required=False, allow_null=True
)
subscriptions_channel_size = serializers.IntegerField(
required=False, allow_null=True
)
subscriptions_live_channel_size = serializers.IntegerField(
required=False, allow_null=True
)
subscriptions_shorts_channel_size = serializers.IntegerField(
required=False, allow_null=True
)
class ChannelSerializer(serializers.Serializer):
"""serialize channel"""
channel_id = serializers.CharField()
channel_active = serializers.BooleanField()
channel_banner_url = serializers.CharField()
channel_thumb_url = serializers.CharField()
channel_tvart_url = serializers.CharField()
channel_description = serializers.CharField()
channel_last_refresh = serializers.CharField()
channel_name = serializers.CharField()
channel_overwrites = ChannelOverwriteSerializer(required=False)
channel_subs = serializers.IntegerField()
channel_subscribed = serializers.BooleanField()
channel_tags = serializers.ListField(child=serializers.CharField())
channel_views = serializers.IntegerField()
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)
class ChannelListSerializer(serializers.Serializer):
"""serialize channel list"""
data = ChannelSerializer(many=True)
paginate = PaginationSerializer()
class ChannelListQuerySerializer(serializers.Serializer):
"""serialize list query"""
filter = serializers.ChoiceField(choices=["subscribed"], required=False)
class ChannelUpdateSerializer(serializers.Serializer):
"""update channel"""
channel_subscribed = serializers.BooleanField(required=False)
channel_overwrites = ChannelOverwriteSerializer(required=False)
class ChannelAggBucketSerializer(serializers.Serializer):
"""serialize channel agg bucket"""
value = serializers.IntegerField()
value_str = serializers.CharField(required=False)
class ChannelAggSerializer(serializers.Serializer):
"""serialize channel aggregation"""
total_items = ChannelAggBucketSerializer()
total_size = ChannelAggBucketSerializer()
total_duration = ChannelAggBucketSerializer()
class ChannelNavSerializer(serializers.Serializer):
"""serialize channel navigation"""
has_pending = serializers.BooleanField()
has_playlists = serializers.BooleanField()
has_videos = serializers.BooleanField()
has_streams = serializers.BooleanField()
has_shorts = serializers.BooleanField()
class ChannelSearchQuerySerializer(serializers.Serializer):
"""serialize query parameters for searching"""
q = serializers.CharField()

4
backend/channel/src/index.py
View File

@ -23,7 +23,7 @@ class YoutubeChannel(YouTubeItem):
index_name = "ta_channel"
yt_base = "https://www.youtube.com/channel/"
yt_obs = {
"playlist_items": "1,0",
"playlist_items": "0,0",
"skip_download": True,
}
@ -115,7 +115,7 @@ class YoutubeChannel(YouTubeItem):
"channel_tvart_url": False,
"channel_id": self.youtube_id,
"channel_subscribed": False,
"channel_tags": False,
"channel_tags": [],
"channel_description": False,
"channel_thumb_url": False,
"channel_views": 0,

181
backend/channel/views.py
View File

@ -1,10 +1,25 @@
"""all channel API views"""
from channel.serializers import (
ChannelAggSerializer,
ChannelListQuerySerializer,
ChannelListSerializer,
ChannelNavSerializer,
ChannelSearchQuerySerializer,
ChannelSerializer,
ChannelUpdateSerializer,
)
from channel.src.index import YoutubeChannel, channel_overwrites
from channel.src.nav import ChannelNav
from common.serializers import ErrorResponseSerializer
from common.src.urlparser import Parser
from common.views_base import AdminWriteOnly, ApiBaseView
from download.src.subscriptions import ChannelSubscription
from drf_spectacular.utils import (
OpenApiParameter,
OpenApiResponse,
extend_schema,
)
from rest_framework.response import Response
from task.tasks import index_channel_playlists, subscribe_to
@ -19,26 +34,38 @@ class ChannelApiListView(ApiBaseView):
valid_filter = ["subscribed"]
permission_classes = [AdminWriteOnly]
@extend_schema(
responses={
200: OpenApiResponse(ChannelListSerializer()),
},
parameters=[
OpenApiParameter(
name="filter",
description="Filter by Subscribed",
type=ChannelListQuerySerializer(),
),
],
)
def get(self, request):
"""get request"""
self.data.update(
{"sort": [{"channel_name.keyword": {"order": "asc"}}]}
)
query_filter = request.GET.get("filter", False)
must_list = []
if query_filter:
if query_filter not in self.valid_filter:
message = f"invalid url query filter: {query_filter}"
print(message)
return Response({"message": message}, status=400)
serializer = ChannelListQuerySerializer(data=request.query_params)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
must_list = []
query_filter = validated_data.get("filter")
if query_filter:
must_list.append({"term": {"channel_subscribed": {"value": True}}})
self.data["query"] = {"bool": {"must": must_list}}
self.get_document_list(request)
serializer = ChannelListSerializer(self.response)
return Response(self.response)
return Response(serializer.data)
def post(self, request):
"""subscribe/unsubscribe to list of channels"""
@ -81,53 +108,79 @@ class ChannelApiView(ApiBaseView):
search_base = "ta_channel/_doc/"
permission_classes = [AdminWriteOnly]
@extend_schema(
responses={
200: OpenApiResponse(ChannelSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="Channel not found"
),
}
)
def get(self, request, channel_id):
# pylint: disable=unused-argument
"""get request"""
"""get channel detail"""
self.get_document(channel_id)
return Response(self.response, status=self.status_code)
if not self.response:
error = ErrorResponseSerializer({"error": "channel not found"})
return Response(error.data, status=404)
response_serializer = ChannelSerializer(self.response)
return Response(response_serializer.data, status=self.status_code)
@extend_schema(
request=ChannelUpdateSerializer(),
responses={
200: OpenApiResponse(ChannelUpdateSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
404: OpenApiResponse(
ErrorResponseSerializer(), description="Channel not found"
),
},
)
def post(self, request, channel_id):
"""modify channel overwrites"""
"""modify channel"""
self.get_document(channel_id)
if not self.response["data"]:
return Response({"error": "channel not found"}, status=404)
if not self.response:
error = ErrorResponseSerializer({"error": "channel not found"})
return Response(error.data, status=404)
data = request.data
subscribed = data.get("channel_subscribed")
serializer = ChannelUpdateSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
subscribed = validated_data.get("channel_subscribed")
if subscribed is not None:
channel_sub = ChannelSubscription()
json_data = channel_sub.change_subscribe(channel_id, subscribed)
return Response(json_data, status=200)
ChannelSubscription().change_subscribe(channel_id, subscribed)
if "channel_overwrites" not in data:
return Response({"error": "invalid payload"}, status=400)
overwrites = data["channel_overwrites"]
try:
json_data = channel_overwrites(channel_id, overwrites)
overwrites = validated_data.get("channel_overwrites")
if overwrites:
channel_overwrites(channel_id, overwrites)
if overwrites.get("index_playlists"):
index_channel_playlists.delay(channel_id)
except ValueError as err:
return Response({"error": str(err)}, status=400)
return Response(json_data, status=200)
return Response(serializer.data, status=200)
@extend_schema(
responses={
204: OpenApiResponse(description="Channel deleted"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="Channel not found"
),
},
)
def delete(self, request, channel_id):
# pylint: disable=unused-argument
"""delete channel"""
message = {"channel": channel_id}
try:
YoutubeChannel(channel_id).delete_channel()
status_code = 200
message.update({"state": "delete"})
return Response(status=204)
except FileNotFoundError:
status_code = 404
message.update({"state": "not found"})
pass
return Response(message, status=status_code)
error = ErrorResponseSerializer({"error": "channel not found"})
return Response(error.data, status=404)
class ChannelAggsApiView(ApiBaseView):
@ -137,8 +190,13 @@ class ChannelAggsApiView(ApiBaseView):
search_base = "ta_video/_search"
@extend_schema(
responses={
200: OpenApiResponse(ChannelAggSerializer()),
},
)
def get(self, request, channel_id):
"""get aggs"""
"""get channel aggregations"""
self.data.update(
{
"query": {
@ -152,8 +210,9 @@ class ChannelAggsApiView(ApiBaseView):
}
)
self.get_aggs()
serializer = ChannelAggSerializer(self.response)
return Response(self.response)
return Response(serializer.data)
class ChannelNavApiView(ApiBaseView):
@ -161,11 +220,17 @@ class ChannelNavApiView(ApiBaseView):
GET: get channel nav
"""
@extend_schema(
responses={
200: OpenApiResponse(ChannelNavSerializer()),
},
)
def get(self, request, channel_id):
"""get nav"""
"""get navigation"""
nav = ChannelNav(channel_id).get_nav()
return Response(nav)
serializer = ChannelNavSerializer(nav)
return Response(serializer.data)
class ChannelApiSearchView(ApiBaseView):
@ -175,10 +240,31 @@ class ChannelApiSearchView(ApiBaseView):
search_base = "ta_channel/_doc/"
@extend_schema(
responses={
200: OpenApiResponse(ChannelSerializer()),
400: OpenApiResponse(description="Bad Request"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="Channel not found"
),
},
parameters=[
OpenApiParameter(
name="q",
description="Search query string",
required=True,
type=str,
),
],
)
def get(self, request):
"""handle get request, search with s parameter"""
"""search for local channel ID"""
query = request.GET.get("q")
serializer = ChannelSearchQuerySerializer(data=request.query_params)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
query = validated_data.get("q")
if not query:
message = "missing expected q parameter"
return Response({"message": message, "data": False}, status=400)
@ -186,13 +272,16 @@ class ChannelApiSearchView(ApiBaseView):
try:
parsed = Parser(query).parse()[0]
except (ValueError, IndexError, AttributeError):
message = f"channel not found: {query}"
return Response({"message": message, "data": False}, status=404)
error = ErrorResponseSerializer(
{"error": f"channel not found: {query}"}
)
return Response(error.data, status=404)
if not parsed["type"] == "channel":
message = "expected type channel"
return Response({"message": message, "data": False}, status=400)
error = ErrorResponseSerializer({"error": "expected channel data"})
return Response(error.data, status=400)
self.get_document(parsed["url"])
serializer = ChannelSerializer(self.response)
return Response(self.response, status=self.status_code)
return Response(serializer.data, status=self.status_code)

142
backend/common/serializers.py Normal file
View File

@ -0,0 +1,142 @@
"""common serializers"""
# pylint: disable=abstract-method
from rest_framework import serializers
class ValidateUnknownFieldsMixin:
"""
Mixin to validate and reject unknown fields in a serializer.
"""
def to_internal_value(self, data):
"""check expected keys"""
allowed_fields = set(self.fields.keys())
input_fields = set(data.keys())
# Find unknown fields
unknown_fields = input_fields - allowed_fields
if unknown_fields:
raise serializers.ValidationError(
{"error": f"Unknown fields: {', '.join(unknown_fields)}"}
)
return super().to_internal_value(data)
class ErrorResponseSerializer(serializers.Serializer):
"""error message"""
error = serializers.CharField()
class PaginationSerializer(serializers.Serializer):
"""serialize paginate response"""
page_size = serializers.IntegerField()
page_from = serializers.IntegerField()
prev_pages = serializers.ListField(
child=serializers.IntegerField(), allow_null=True
)
current_page = serializers.IntegerField()
max_hits = serializers.BooleanField()
params = serializers.CharField()
last_page = serializers.IntegerField()
next_pages = serializers.ListField(
child=serializers.IntegerField(), allow_null=True
)
total_hits = serializers.IntegerField()
class AsyncTaskResponseSerializer(serializers.Serializer):
"""serialize new async task"""
message = serializers.CharField(required=False)
task_id = serializers.CharField()
status = serializers.CharField(required=False)
filename = serializers.CharField(required=False)
class NotificationSerializer(serializers.Serializer):
"""serialize notification messages"""
id = serializers.CharField()
title = serializers.CharField()
group = serializers.CharField()
api_start = serializers.BooleanField()
api_stop = serializers.BooleanField()
level = serializers.ChoiceField(choices=["info", "error"])
messages = serializers.ListField(child=serializers.CharField())
progress = serializers.FloatField(required=False)
class NotificationQueryFilterSerializer(serializers.Serializer):
"""serialize notification query filter"""
filter = serializers.ChoiceField(
choices=["download", "settings", "channel"], required=False
)
class PingUpdateSerializer(serializers.Serializer):
"""serialize update notification"""
status = serializers.BooleanField()
version = serializers.CharField()
is_breaking = serializers.BooleanField()
class PingSerializer(serializers.Serializer):
"""serialize ping response"""
response = serializers.ChoiceField(choices=["pong"])
user = serializers.IntegerField()
version = serializers.CharField()
ta_update = PingUpdateSerializer(required=False)
class WatchedDataSerializer(serializers.Serializer):
"""mark as watched serializer"""
id = serializers.CharField()
is_watched = serializers.BooleanField()
class RefreshQuerySerializer(serializers.Serializer):
"""refresh query filtering"""
type = serializers.ChoiceField(
choices=["video", "channel", "playlist"], required=False
)
id = serializers.CharField(required=False)
class RefreshResponseSerializer(serializers.Serializer):
"""serialize refresh response"""
state = serializers.ChoiceField(
choices=["running", "queued", "empty", False]
)
total_queued = serializers.IntegerField()
in_queue_name = serializers.CharField(required=False)
class RefreshAddQuerySerializer(serializers.Serializer):
"""serialize add to refresh queue"""
extract_videos = serializers.BooleanField(required=False)
class RefreshAddDataSerializer(serializers.Serializer):
"""add to refresh queue serializer"""
video = serializers.ListField(
child=serializers.CharField(), required=False
)
channel = serializers.ListField(
child=serializers.CharField(), required=False
)
playlist = serializers.ListField(
child=serializers.CharField(), required=False
)

2
backend/common/src/env_settings.py
View File

@ -25,6 +25,7 @@ class EnvironmentSettings:
HOST_UID: int = int(environ.get("HOST_UID", False))
HOST_GID: int = int(environ.get("HOST_GID", False))
ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
DISABLE_STATIC_AUTH: bool = bool(environ.get("DISABLE_STATIC_AUTH"))
TZ: str = str(environ.get("TZ", "UTC"))
TA_PORT: int = int(environ.get("TA_PORT", False))
TA_BACKEND_PORT: int = int(environ.get("TA_BACKEND_PORT", False))
@ -73,6 +74,7 @@ class EnvironmentSettings:
HOST_GID: {self.HOST_GID}
TZ: {self.TZ}
ENABLE_CAST: {self.ENABLE_CAST}
DISABLE_STATIC_AUTH: {self.DISABLE_STATIC_AUTH}
TA_PORT: {self.TA_PORT}
TA_BACKEND_PORT: {self.TA_BACKEND_PORT}
TA_USERNAME: {self.TA_USERNAME}

2
backend/common/src/index_generic.py
View File

@ -106,7 +106,7 @@ class Pagination:
page_get = self.page_get
page_from = 0
if page_get in [0, 1]:
prev_pages = False
prev_pages = None
elif page_get > 1:
page_from = (page_get - 1) * self.page_size
prev_pages = [

5
backend/common/src/search_processor.py
View File

@ -115,6 +115,8 @@ class SearchProcess:
video_dict["subtitles"][idx][
"media_url"
] = f"{media_root}/{url}"
else:
video_dict["subtitles"] = []
video_dict.update(
{
@ -138,6 +140,9 @@ class SearchProcess:
}
)
if "playlist" not in video_dict:
video_dict["playlist"] = []
return dict(sorted(video_dict.items()))
@staticmethod

123
backend/common/views.py
View File

@ -2,10 +2,23 @@
from appsettings.src.config import ReleaseVersion
from appsettings.src.reindex import ReindexProgress
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
NotificationQueryFilterSerializer,
NotificationSerializer,
PingSerializer,
RefreshAddDataSerializer,
RefreshAddQuerySerializer,
RefreshQuerySerializer,
RefreshResponseSerializer,
WatchedDataSerializer,
)
from common.src.searching import SearchForm
from common.src.ta_redis import RedisArchivist
from common.src.watched import WatchState
from common.views_base import AdminOnly, ApiBaseView
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.response import Response
from task.tasks import check_reindex
@ -16,6 +29,9 @@ class PingView(ApiBaseView):
"""
@staticmethod
@extend_schema(
responses={200: OpenApiResponse(PingSerializer())},
)
def get(request):
"""get pong"""
data = {
@ -24,7 +40,8 @@ class PingView(ApiBaseView):
"version": ReleaseVersion().get_local_version(),
"ta_update": ReleaseVersion().get_update(),
}
return Response(data)
serializer = PingSerializer(data)
return Response(serializer.data)
class RefreshView(ApiBaseView):
@ -35,30 +52,69 @@ class RefreshView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(RefreshResponseSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
parameters=[RefreshQuerySerializer()],
)
def get(self, request):
"""handle get request"""
request_type = request.GET.get("type")
request_id = request.GET.get("id")
"""get refresh status"""
query_serializer = RefreshQuerySerializer(data=request.query_params)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
request_type = validated_query.get("type")
request_id = validated_query.get("id")
if request_id and not request_type:
return Response({"status": "Bad Request"}, status=400)
error = ErrorResponseSerializer(
{"error": "specified id also needs type"}
)
return Response(error.data, status=400)
try:
progress = ReindexProgress(
request_type=request_type, request_id=request_id
).get_progress()
except ValueError:
return Response({"status": "Bad Request"}, status=400)
error = ErrorResponseSerializer({"error": "bad request"})
return Response(error.data, status=400)
return Response(progress)
response_serializer = RefreshResponseSerializer(progress)
return Response(response_serializer.data)
@extend_schema(
request=RefreshAddDataSerializer(),
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
},
parameters=[RefreshAddQuerySerializer()],
)
def post(self, request):
"""handle post request"""
data = request.data
extract_videos = bool(request.GET.get("extract_videos", False))
check_reindex.delay(data=data, extract_videos=extract_videos)
"""add to reindex queue"""
query_serializer = RefreshAddQuerySerializer(data=request.query_params)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
return Response(data)
data_serializer = RefreshAddDataSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
extract_videos = validated_query.get("extract_videos")
task = check_reindex.delay(
data=validated_data, extract_videos=extract_videos
)
message = {
"message": "reindex task started",
"task_id": task.id,
}
serializer = AsyncTaskResponseSerializer(message)
return Response(serializer.data)
class WatchedView(ApiBaseView):
@ -66,17 +122,31 @@ class WatchedView(ApiBaseView):
POST: change watched state of video, channel or playlist
"""
@extend_schema(
request=WatchedDataSerializer(),
responses={
200: OpenApiResponse(WatchedDataSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(self, request):
"""change watched state"""
youtube_id = request.data.get("id")
is_watched = request.data.get("is_watched")
data_serializer = WatchedDataSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
youtube_id = validated_data.get("id")
is_watched = validated_data.get("is_watched")
if not youtube_id or is_watched is None:
message = {"message": "missing id or is_watched"}
return Response(message, status=400)
error = ErrorResponseSerializer(
{"error": "missing id or is_watched"}
)
return Response(error.data, status=400)
WatchState(youtube_id, is_watched, request.user.id).change()
return Response({"message": "success"}, status=200)
return Response(data_serializer.data)
class SearchView(ApiBaseView):
@ -106,11 +176,26 @@ class NotificationView(ApiBaseView):
valid_filters = ["download", "settings", "channel"]
@extend_schema(
responses={
200: OpenApiResponse(NotificationSerializer(many=True)),
},
parameters=[NotificationQueryFilterSerializer],
)
def get(self, request):
"""get all notifications"""
query_serializer = NotificationQueryFilterSerializer(
data=request.query_params
)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
filter_by = validated_query.get("filter")
query = "message"
filter_by = request.GET.get("filter", None)
if filter_by in self.valid_filters:
query = f"{query}:{filter_by}"
return Response(RedisArchivist().list_items(query))
notifications = RedisArchivist().list_items(query)
response_serializer = NotificationSerializer(notifications, many=True)
return Response(response_serializer.data)

14
backend/common/views_base.py
View File

@ -1,7 +1,5 @@
"""base classes to inherit from"""
from appsettings.src.config import AppConfig
from common.src.env_settings import EnvironmentSettings
from common.src.es_connect import ElasticWrap
from common.src.index_generic import Pagination
from common.src.search_processor import SearchProcess, process_aggs
@ -45,13 +43,7 @@ class ApiBaseView(APIView):
def __init__(self):
super().__init__()
self.response = {
"data": False,
"config": {
"enable_cast": EnvironmentSettings.ENABLE_CAST,
"downloads": AppConfig().config["downloads"],
},
}
self.response = {}
self.data = {"query": {"match_all": {}}}
self.status_code = False
self.context = False
@ -62,12 +54,12 @@ class ApiBaseView(APIView):
path = f"{self.search_base}{document_id}"
response, status_code = ElasticWrap(path).get()
try:
self.response["data"] = SearchProcess(
self.response = SearchProcess(
response, match_video_user_progress=progress_match
).process()
except KeyError:
print(f"item not found: {document_id}")
self.response["data"] = False
self.status_code = status_code
def initiate_pagination(self, request):

36
backend/config/management/commands/ta_change_password.py Normal file
View File

@ -0,0 +1,36 @@
"""change user password"""
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
User = get_user_model()
class Command(BaseCommand):
"""change password"""
help = "Change Password of user"
def add_arguments(self, parser):
parser.add_argument("username", type=str)
parser.add_argument("password", type=str)
def handle(self, *args, **kwargs):
"""entry point"""
username = kwargs["username"]
new_password = kwargs["password"]
self.stdout.write(f"Changing password for user '{username}'")
try:
user = User.objects.get(name=username)
except User.DoesNotExist as err:
message = f"Username '{username}' does not exist. "
message += "Available username(s) are:\n"
message += ", ".join([i.name for i in User.objects.all()])
raise CommandError(message) from err
user.set_password(new_password)
user.save()
self.stdout.write(
self.style.SUCCESS(f" ✓ updated password for user '{username}'")
)

18
backend/config/management/commands/ta_envcheck.py
View File

@ -168,25 +168,35 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS(message))
def _enable_cast_overwrite(self):
"""cast workaround, remove auth for static files in nginx"""
"""enable cast env var"""
self.stdout.write("[6] check ENABLE_CAST overwrite")
overwrite = EnvironmentSettings.ENABLE_CAST
if not overwrite:
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
return
def _disable_static_auth(self):
"""cast workaround, remove auth for static files in nginx"""
self.stdout.write("[7] check DISABLE_STATIC_AUTH overwrite")
overwrite = EnvironmentSettings.DISABLE_STATIC_AUTH
if not overwrite:
self.stdout.write(
self.style.SUCCESS(" DISABLE_STATIC_AUTH is not set")
)
return
regex = re.compile(r"[^\S\r\n]*auth_request /api/ping/;\n")
changed = file_overwrite(NGINX, regex, "")
if changed:
message = " ✓ process nginx to enable Cast"
message = " ✓ process nginx to disable static auth"
else:
message = " ✓ Cast is already enabled in nginx"
message = " ✓ static auth is already disabled in nginx"
self.stdout.write(self.style.SUCCESS(message))
def _create_superuser(self):
"""create superuser if not exist"""
self.stdout.write("[7] create superuser")
self.stdout.write("[8] create superuser")
is_created = Account.objects.filter(is_superuser=True)
if is_created:
message = " superuser already created"

276
backend/config/management/commands/ta_startup.py
View File

@ -42,7 +42,6 @@ class Command(BaseCommand):
def handle(self, *args, **options):
"""run all commands"""
self.stdout.write(TOPIC)
self._mig_app_settings()
self._make_folders()
self._clear_redis_keys()
self._clear_tasks()
@ -50,9 +49,113 @@ class Command(BaseCommand):
self._version_check()
self._index_setup()
self._snapshot_check()
self._mig_app_settings()
self._create_default_schedules()
self._update_schedule_tz()
self._init_app_config()
self._mig_channel_tags()
self._mig_video_channel_tags()
def _make_folders(self):
"""make expected cache folders"""
self.stdout.write("[1] create expected cache folders")
folders = [
"backup",
"channels",
"download",
"import",
"playlists",
"videos",
]
cache_dir = EnvironmentSettings.CACHE_DIR
for folder in folders:
folder_path = os.path.join(cache_dir, folder)
os.makedirs(folder_path, exist_ok=True)
self.stdout.write(self.style.SUCCESS(" ✓ expected folders created"))
def _clear_redis_keys(self):
"""make sure there are no leftover locks or keys set in redis"""
self.stdout.write("[2] clear leftover keys in redis")
all_keys = [
"dl_queue_id",
"dl_queue",
"downloading",
"manual_import",
"reindex",
"rescan",
"run_backup",
"startup_check",
"reindex:ta_video",
"reindex:ta_channel",
"reindex:ta_playlist",
]
redis_con = RedisArchivist()
has_changed = False
for key in all_keys:
if redis_con.del_message(key):
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared key {key}")
)
has_changed = True
if not has_changed:
self.stdout.write(self.style.SUCCESS(" no keys found"))
def _clear_tasks(self):
"""clear tasks and messages"""
self.stdout.write("[3] clear task leftovers")
TaskManager().fail_pending()
redis_con = RedisArchivist()
to_delete = redis_con.list_keys("message:")
if to_delete:
for key in to_delete:
redis_con.del_message(key)
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {len(to_delete)} messages")
)
def _clear_dl_cache(self):
"""clear leftover files from dl cache"""
self.stdout.write("[4] clear leftover files from dl cache")
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
if leftover_files:
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
)
else:
self.stdout.write(self.style.SUCCESS(" no files found"))
def _version_check(self):
"""remove new release key if updated now"""
self.stdout.write("[5] check for first run after update")
new_version = ReleaseVersion().is_updated()
if new_version:
self.stdout.write(
self.style.SUCCESS(f" ✓ update to {new_version} completed")
)
else:
self.stdout.write(self.style.SUCCESS(" no new update found"))
version_task = CustomPeriodicTask.objects.filter(name="version_check")
if not version_task.exists():
return
if not version_task.first().last_run_at:
self.style.SUCCESS(" ✓ send initial version check task")
version_check.delay()
def _index_setup(self):
"""migration: validate index mappings"""
self.stdout.write("[6] validate index mappings")
ElasitIndexWrap().setup()
def _snapshot_check(self):
"""migration setup snapshots"""
self.stdout.write("[7] setup snapshots")
ElasticSnapshot().setup()
def _mig_app_settings(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate application settings"""
@ -87,110 +190,9 @@ class Command(BaseCommand):
sleep(60)
raise CommandError(message)
def _make_folders(self):
"""make expected cache folders"""
self.stdout.write("[2] create expected cache folders")
folders = [
"backup",
"channels",
"download",
"import",
"playlists",
"videos",
]
cache_dir = EnvironmentSettings.CACHE_DIR
for folder in folders:
folder_path = os.path.join(cache_dir, folder)
os.makedirs(folder_path, exist_ok=True)
self.stdout.write(self.style.SUCCESS(" ✓ expected folders created"))
def _clear_redis_keys(self):
"""make sure there are no leftover locks or keys set in redis"""
self.stdout.write("[3] clear leftover keys in redis")
all_keys = [
"dl_queue_id",
"dl_queue",
"downloading",
"manual_import",
"reindex",
"rescan",
"run_backup",
"startup_check",
"reindex:ta_video",
"reindex:ta_channel",
"reindex:ta_playlist",
]
redis_con = RedisArchivist()
has_changed = False
for key in all_keys:
if redis_con.del_message(key):
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared key {key}")
)
has_changed = True
if not has_changed:
self.stdout.write(self.style.SUCCESS(" no keys found"))
def _clear_tasks(self):
"""clear tasks and messages"""
self.stdout.write("[4] clear task leftovers")
TaskManager().fail_pending()
redis_con = RedisArchivist()
to_delete = redis_con.list_keys("message:")
if to_delete:
for key in to_delete:
redis_con.del_message(key)
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {len(to_delete)} messages")
)
def _clear_dl_cache(self):
"""clear leftover files from dl cache"""
self.stdout.write("[5] clear leftover files from dl cache")
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
if leftover_files:
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
)
else:
self.stdout.write(self.style.SUCCESS(" no files found"))
def _version_check(self):
"""remove new release key if updated now"""
self.stdout.write("[6] check for first run after update")
new_version = ReleaseVersion().is_updated()
if new_version:
self.stdout.write(
self.style.SUCCESS(f" ✓ update to {new_version} completed")
)
else:
self.stdout.write(self.style.SUCCESS(" no new update found"))
version_task = CustomPeriodicTask.objects.filter(name="version_check")
if not version_task.exists():
return
if not version_task.first().last_run_at:
self.style.SUCCESS(" ✓ send initial version check task")
version_check.delay()
def _index_setup(self):
"""migration: validate index mappings"""
self.stdout.write("[7] validate index mappings")
ElasitIndexWrap().setup()
def _snapshot_check(self):
"""migration setup snapshots"""
self.stdout.write("[8] setup snapshots")
ElasticSnapshot().setup()
def _create_default_schedules(self) -> None:
"""create default schedules for new installations"""
self.stdout.write("[9] create initial schedules")
self.stdout.write("[8] create initial schedules")
init_has_run = CustomPeriodicTask.objects.filter(
name="version_check"
).exists()
@ -241,7 +243,7 @@ class Command(BaseCommand):
def _update_schedule_tz(self) -> None:
"""update timezone for Schedule instances"""
self.stdout.write("[10] validate schedules TZ")
self.stdout.write("[9] validate schedules TZ")
tz = EnvironmentSettings.TZ
to_update = CrontabSchedule.objects.exclude(timezone=tz)
@ -259,7 +261,7 @@ class Command(BaseCommand):
def _init_app_config(self) -> None:
"""init default app config to ES"""
self.stdout.write("[11] Check AppConfig")
self.stdout.write("[10] Check AppConfig")
try:
_ = AppConfig().config
self.stdout.write(
@ -280,3 +282,67 @@ class Command(BaseCommand):
self.stdout.write(
self.style.SUCCESS(f" Status code: {status_code}")
)
def _mig_channel_tags(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate incorrect data types"""
self.stdout.write("[MIGRATION] fix incorrect channel tags types")
path = "ta_channel/_update_by_query"
data = {
"query": {"match": {"channel_tags": False}},
"script": {
"source": "ctx._source.channel_tags = []",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(f" ✓ fixed {updated} channel tags")
)
else:
self.stdout.write(
self.style.SUCCESS(" no channel tags needed fixing")
)
return
message = " 🗙 failed to fix channel tags"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_video_channel_tags(self) -> None:
"""update from v0.4.13 to v0.5.0, migrate incorrect data types"""
self.stdout.write("[MIGRATION] fix incorrect video channel tags types")
path = "ta_video/_update_by_query"
data = {
"query": {"match": {"channel.channel_tags": False}},
"script": {
"source": "ctx._source.channel.channel_tags = []",
"lang": "painless",
},
}
response, status_code = ElasticWrap(path).post(data)
if status_code in [200, 201]:
updated = response.get("updated")
if updated:
self.stdout.write(
self.style.SUCCESS(
f" ✓ fixed {updated} video channel tags"
)
)
else:
self.stdout.write(
self.style.SUCCESS(
" no video channel tags needed fixing"
)
)
return
message = " 🗙 failed to fix video channel tags"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)

13
backend/config/settings.py
View File

@ -59,6 +59,7 @@ INSTALLED_APPS = [
"django.contrib.humanize",
"rest_framework",
"rest_framework.authtoken",
"drf_spectacular",
"common",
"video",
"channel",
@ -295,3 +296,15 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
# TA application settings
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
TA_VERSION = "v0.5.0-unstable"
# API
REST_FRAMEWORK = {
"DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
}
SPECTACULAR_SETTINGS = {
"TITLE": "Tube Archivist API",
"DESCRIPTION": "API documentation for Tube Archivist backend.",
"VERSION": TA_VERSION,
"SERVE_INCLUDE_SCHEMA": False,
}

7
backend/config/urls.py
View File

@ -16,6 +16,7 @@ Including another URLconf
from django.contrib import admin
from django.urls import include, path
from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
urlpatterns = [
path("api/", include("common.urls")),
@ -27,5 +28,11 @@ urlpatterns = [
path("api/appsettings/", include("appsettings.urls")),
path("api/stats/", include("stats.urls")),
path("api/user/", include("user.urls")),
path("api/schema/", SpectacularAPIView.as_view(), name="schema"),
path(
"api/docs/",
SpectacularSwaggerView.as_view(url_name="schema"),
name="swagger-ui",
),
path("admin/", admin.site.urls),
]

93
backend/download/serializers.py Normal file
View File

@ -0,0 +1,93 @@
"""download serializers"""
# pylint: disable=abstract-method
from common.serializers import PaginationSerializer, ValidateUnknownFieldsMixin
from rest_framework import serializers
from video.src.constants import VideoTypeEnum
class DownloadItemSerializer(serializers.Serializer):
"""serialize download item"""
auto_start = serializers.BooleanField()
channel_id = serializers.CharField()
channel_indexed = serializers.BooleanField()
channel_name = serializers.CharField()
duration = serializers.CharField()
published = serializers.CharField()
status = serializers.ChoiceField(choices=["pending", "ignore"])
timestamp = serializers.IntegerField()
title = serializers.CharField()
vid_thumb_url = serializers.CharField()
vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values())
youtube_id = serializers.CharField()
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)
class DownloadListSerializer(serializers.Serializer):
"""serialize download list"""
data = DownloadItemSerializer(many=True)
paginate = PaginationSerializer()
class DownloadListQuerySerializer(
ValidateUnknownFieldsMixin, serializers.Serializer
):
"""serialize query params for download list"""
filter = serializers.ChoiceField(
choices=["pending", "ignore"], required=False
)
channel = serializers.CharField(required=False, help_text="channel ID")
class DownloadListQueueDeleteQuerySerializer(serializers.Serializer):
"""serialize bulk delete download queue query string"""
filter = serializers.ChoiceField(choices=["pending", "ignore"])
class AddDownloadItemSerializer(serializers.Serializer):
"""serialize single item to add"""
youtube_id = serializers.CharField()
status = serializers.ChoiceField(choices=["pending"])
class AddToDownloadListSerializer(serializers.Serializer):
"""serialize add to download queue data"""
data = AddDownloadItemSerializer(many=True)
class AddToDownloadQuerySerializer(serializers.Serializer):
"""add to queue query serializer"""
autostart = serializers.BooleanField(required=False)
class DownloadQueueItemUpdateSerializer(serializers.Serializer):
"""update single download queue item"""
status = serializers.ChoiceField(
choices=["pending", "ignore", "ignore-force", "priority"]
)
class DownloadAggBucketSerializer(serializers.Serializer):
"""serialize bucket"""
key = serializers.ListField(child=serializers.CharField())
key_as_string = serializers.CharField()
doc_count = serializers.IntegerField()
class DownloadAggsSerializer(serializers.Serializer):
"""serialize download channel bucket aggregations"""
doc_count_error_upper_bound = serializers.IntegerField()
sum_other_doc_count = serializers.IntegerField()
buckets = DownloadAggBucketSerializer(many=True)

2
backend/download/src/queue.py
View File

@ -322,7 +322,7 @@ class PendingList(PendingIndex):
duration = vid.get("duration")
if duration and isinstance(duration, int):
if duration > 60:
if duration > 3 * 60:
return False
return is_shorts(vid["id"])

8
backend/download/src/yt_dlp_base.py
View File

@ -134,7 +134,7 @@ class CookieHandler:
"""set cookie str and activate in config"""
cookie_clean = cookie.strip("\x00")
RedisArchivist().set_message("cookie", cookie_clean, save=True)
AppConfig().update_config({"downloads.cookie_import": True})
AppConfig().update_config({"downloads": {"cookie_import": True}})
self.config["downloads"]["cookie_import"] = True
print("[cookie]: activated and stored in Redis")
@ -143,7 +143,7 @@ class CookieHandler:
"""revoke cookie"""
RedisArchivist().del_message("cookie")
RedisArchivist().del_message("cookie:valid")
AppConfig().update_config({"downloads.cookie_import": False})
AppConfig().update_config({"downloads": {"cookie_import": False}})
print("[cookie]: revoked")
def validate(self):
@ -211,9 +211,9 @@ class POTokenHandler:
def set_token(self, new_token: str) -> None:
"""set new PO token"""
RedisArchivist().set_message(self.REDIS_KEY, new_token)
AppConfig().update_config({"downloads.potoken": True})
AppConfig().update_config({"downloads": {"potoken": True}})
def revoke_token(self) -> None:
"""revoke token"""
RedisArchivist().del_message(self.REDIS_KEY)
AppConfig().update_config({"downloads.potoken": False})
AppConfig().update_config({"downloads": {"potoken": False}})

275
backend/download/views.py
View File

@ -1,7 +1,22 @@
"""all download API views"""
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
)
from common.views_base import AdminOnly, ApiBaseView
from download.serializers import (
AddToDownloadListSerializer,
AddToDownloadQuerySerializer,
DownloadAggsSerializer,
DownloadItemSerializer,
DownloadListQuerySerializer,
DownloadListQueueDeleteQuerySerializer,
DownloadListSerializer,
DownloadQueueItemUpdateSerializer,
)
from download.src.queue import PendingInteract
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.response import Response
from task.tasks import download_pending, extrac_dl
@ -17,8 +32,14 @@ class DownloadApiListView(ApiBaseView):
valid_filter = ["pending", "ignore"]
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(DownloadListSerializer()),
},
parameters=[DownloadListQuerySerializer()],
)
def get(self, request):
"""get request"""
"""get download queue list"""
query_filter = request.GET.get("filter", False)
self.data.update(
{
@ -29,16 +50,16 @@ class DownloadApiListView(ApiBaseView):
}
)
must_list = []
if query_filter:
if query_filter not in self.valid_filter:
message = f"invalid url query filter: {query_filter}"
print(message)
return Response({"message": message}, status=400)
serializer = DownloadListQuerySerializer(data=request.query_params)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
must_list = []
query_filter = validated_data.get("filter")
if query_filter:
must_list.append({"term": {"status": {"value": query_filter}}})
filter_channel = request.GET.get("channel", False)
filter_channel = validated_data.get("channel")
if filter_channel:
must_list.append(
{"term": {"channel_id": {"value": filter_channel}}}
@ -47,39 +68,169 @@ class DownloadApiListView(ApiBaseView):
self.data["query"] = {"bool": {"must": must_list}}
self.get_document_list(request)
return Response(self.response)
serializer = DownloadListSerializer(self.response)
return Response(serializer.data)
@staticmethod
@extend_schema(
request=AddToDownloadListSerializer(),
parameters=[AddToDownloadQuerySerializer()],
responses={
200: OpenApiResponse(
AsyncTaskResponseSerializer(),
description="New async task started",
),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(request):
"""add list of videos to download queue"""
data = request.data
auto_start = bool(request.GET.get("autostart"))
try:
to_add = data["data"]
except KeyError:
message = "missing expected data key"
print(message)
return Response({"message": message}, status=400)
data_serializer = AddToDownloadListSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
query_serializer = AddToDownloadQuerySerializer(
data=request.query_params
)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
auto_start = validated_query.get("autostart")
print(f"auto_start: {auto_start}")
to_add = validated_data["data"]
pending = [i["youtube_id"] for i in to_add if i["status"] == "pending"]
url_str = " ".join(pending)
extrac_dl.delay(url_str, auto_start=auto_start)
task = extrac_dl.delay(url_str, auto_start=auto_start)
return Response(data)
message = {
"message": "add to queue task started",
"task_id": task.id,
}
response_serializer = AsyncTaskResponseSerializer(message)
return Response(response_serializer.data)
@extend_schema(
parameters=[DownloadListQueueDeleteQuerySerializer()],
responses={
204: OpenApiResponse(description="Download items deleted"),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def delete(self, request):
"""delete download queue"""
query_filter = request.GET.get("filter", False)
if query_filter not in self.valid_filter:
message = f"invalid url query filter: {query_filter}"
print(message)
return Response({"message": message}, status=400)
"""bulk delete download queue items by filter"""
serializer = DownloadListQueueDeleteQuerySerializer(
data=request.query_params
)
serializer.is_valid(raise_exception=True)
validated_query = serializer.validated_data
query_filter = validated_query["filter"]
message = f"delete queue by status: {query_filter}"
print(message)
PendingInteract(status=query_filter).delete_by_status()
return Response({"message": message})
return Response(status=204)
class DownloadApiView(ApiBaseView):
"""resolves to /api/download/<video_id>/
GET: returns metadata dict of an item in the download queue
POST: update status of item to pending or ignore
DELETE: forget from download queue
"""
search_base = "ta_download/_doc/"
valid_status = ["pending", "ignore", "ignore-force", "priority"]
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(DownloadItemSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(),
description="Download item not found",
),
},
)
def get(self, request, video_id):
# pylint: disable=unused-argument
"""get download queue item"""
self.get_document(video_id)
if not self.response:
error = ErrorResponseSerializer(
{"error": "Download item not found"}
)
return Response(error.data, status=404)
response_serializer = DownloadItemSerializer(self.response)
return Response(response_serializer.data, status=self.status_code)
@extend_schema(
request=DownloadQueueItemUpdateSerializer(),
responses={
200: OpenApiResponse(
DownloadQueueItemUpdateSerializer(),
description="Download item update",
),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
404: OpenApiResponse(
ErrorResponseSerializer(),
description="Download item not found",
),
},
)
def post(self, request, video_id):
"""post to video to change status"""
data_serializer = DownloadQueueItemUpdateSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
item_status = validated_data["status"]
if item_status == "ignore-force":
extrac_dl.delay(video_id, status="ignore")
return Response(data_serializer.data)
_, status_code = PendingInteract(video_id).get_item()
if status_code == 404:
error = ErrorResponseSerializer(
{"error": "Download item not found"}
)
return Response(error.data, status=404)
print(f"{video_id}: change status to {item_status}")
PendingInteract(video_id, item_status).update_status()
if item_status == "priority":
download_pending.delay(auto_only=True)
return Response(data_serializer.data)
@staticmethod
@extend_schema(
responses={
204: OpenApiResponse(description="delete download item"),
404: OpenApiResponse(
ErrorResponseSerializer(),
description="Download item not found",
),
},
)
def delete(request, video_id):
# pylint: disable=unused-argument
"""delete single video from queue"""
print(f"{video_id}: delete from queue")
PendingInteract(video_id).delete_item()
return Response(status=204)
class DownloadAggsApiView(ApiBaseView):
@ -90,9 +241,24 @@ class DownloadAggsApiView(ApiBaseView):
search_base = "ta_download/_search"
valid_filter_view = ["ignore", "pending"]
@extend_schema(
parameters=[DownloadListQueueDeleteQuerySerializer()],
responses={
200: OpenApiResponse(DownloadAggsSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
},
)
def get(self, request):
"""get aggs"""
filter_view = request.GET.get("filter")
serializer = DownloadListQueueDeleteQuerySerializer(
data=request.query_params
)
serializer.is_valid(raise_exception=True)
validated_query = serializer.validated_data
filter_view = validated_query.get("filter")
if filter_view:
if filter_view not in self.valid_filter_view:
message = f"invalid filter: {filter_view}"
@ -121,57 +287,6 @@ class DownloadAggsApiView(ApiBaseView):
}
)
self.get_aggs()
serializer = DownloadAggsSerializer(self.response["channel_downloads"])
return Response(self.response)
class DownloadApiView(ApiBaseView):
"""resolves to /api/download/<video_id>/
GET: returns metadata dict of an item in the download queue
POST: update status of item to pending or ignore
DELETE: forget from download queue
"""
search_base = "ta_download/_doc/"
valid_status = ["pending", "ignore", "ignore-force", "priority"]
permission_classes = [AdminOnly]
def get(self, request, video_id):
# pylint: disable=unused-argument
"""get request"""
self.get_document(video_id)
return Response(self.response, status=self.status_code)
def post(self, request, video_id):
"""post to video to change status"""
item_status = request.data.get("status")
if item_status not in self.valid_status:
message = f"{video_id}: invalid status {item_status}"
print(message)
return Response({"message": message}, status=400)
if item_status == "ignore-force":
extrac_dl.delay(video_id, status="ignore")
message = f"{video_id}: set status to ignore"
return Response(request.data)
_, status_code = PendingInteract(video_id).get_item()
if status_code == 404:
message = f"{video_id}: item not found {status_code}"
return Response({"message": message}, status=404)
print(f"{video_id}: change status to {item_status}")
PendingInteract(video_id, item_status).update_status()
if item_status == "priority":
download_pending.delay(auto_only=True)
return Response(request.data)
@staticmethod
def delete(request, video_id):
# pylint: disable=unused-argument
"""delete single video from queue"""
print(f"{video_id}: delete from queue")
PendingInteract(video_id).delete_item()
return Response({"success": True})
return Response(serializer.data)

91
backend/playlist/serializers.py Normal file
View File

@ -0,0 +1,91 @@
"""playlist serializers"""
# pylint: disable=abstract-method
from common.serializers import PaginationSerializer
from rest_framework import serializers
class PlaylistEntrySerializer(serializers.Serializer):
"""serialize single playlist entry"""
youtube_id = serializers.CharField()
title = serializers.CharField()
uploader = serializers.CharField()
idx = serializers.IntegerField()
downloaded = serializers.BooleanField()
class PlaylistSerializer(serializers.Serializer):
"""serialize playlist"""
playlist_active = serializers.BooleanField()
playlist_channel = serializers.CharField()
playlist_channel_id = serializers.CharField()
playlist_description = serializers.CharField()
playlist_entries = PlaylistEntrySerializer(many=True)
playlist_id = serializers.CharField()
playlist_last_refresh = serializers.CharField()
playlist_name = serializers.CharField()
playlist_subscribed = serializers.BooleanField()
playlist_thumbnail = serializers.CharField()
playlist_type = serializers.ChoiceField(choices=["regular", "custom"])
_index = serializers.CharField(required=False)
_score = serializers.IntegerField(required=False)
class PlaylistListSerializer(serializers.Serializer):
"""serialize list of playlists"""
data = PlaylistSerializer(many=True)
paginate = PaginationSerializer()
class PlaylistListQuerySerializer(serializers.Serializer):
"""serialize playlist list query params"""
channel = serializers.CharField(required=False)
subscribed = serializers.BooleanField(required=False)
type = serializers.ChoiceField(
choices=["regular", "custom"], required=False
)
class PlaylistSingleAddSerializer(serializers.Serializer):
"""single item to add"""
playlist_id = serializers.CharField()
playlist_subscribed = serializers.ChoiceField(choices=[True])
class PlaylistBulkAddSerializer(serializers.Serializer):
"""bulk add playlists serializers"""
data = PlaylistSingleAddSerializer(many=True)
class PlaylistSingleUpdate(serializers.Serializer):
"""update state of single playlist"""
playlist_subscribed = serializers.BooleanField()
class PlaylistListCustomPostSerializer(serializers.Serializer):
"""serialize list post custom playlist"""
playlist_name = serializers.CharField()
class PlaylistCustomPostSerializer(serializers.Serializer):
"""serialize playlist custom action"""
action = serializers.ChoiceField(
choices=["create", "remove", "up", "down", "top", "bottom"]
)
video_id = serializers.CharField()
class PlaylistDeleteQuerySerializer(serializers.Serializer):
"""serialize playlist delete query params"""
delete_videos = serializers.BooleanField(required=False)

4
backend/playlist/src/index.py
View File

@ -201,7 +201,7 @@ class YoutubePlaylist(YouTubeItem):
current_idx = all_entries.index(current[0])
if current_idx == 0:
previous_item = False
previous_item = None
else:
previous_item = all_entries[current_idx - 1]
prev_id = previous_item["youtube_id"]
@ -209,7 +209,7 @@ class YoutubePlaylist(YouTubeItem):
previous_item["vid_thumb"] = f"{cache_root}/{prev_thumb_path}"
if current_idx == len(all_entries) - 1:
next_item = False
next_item = None
else:
next_item = all_entries[current_idx + 1]
next_id = next_item["youtube_id"]

7
backend/playlist/src/query_building.py
View File

@ -23,16 +23,15 @@ class QueryBuilder:
must_list = []
channel = self.request_params.get("channel")
if channel:
must_list.append({"match": {"playlist_channel_id": channel[0]}})
must_list.append({"match": {"playlist_channel_id": channel}})
subscribed = self.request_params.get("subscribed")
if subscribed:
subed_bool = subscribed[0] == "true"
must_list.append({"match": {"playlist_subscribed": subed_bool}})
must_list.append({"match": {"playlist_subscribed": subscribed}})
playlist_type = self.request_params.get("type")
if playlist_type:
type_list = self.parse_type(playlist_type[0])
type_list = self.parse_type(playlist_type)
must_list.append(type_list)
query = {"bool": {"must": must_list}}

8
backend/playlist/tests/test_src/test_query_building.py
View File

@ -7,9 +7,9 @@ from playlist.src.query_building import QueryBuilder
def test_build_data():
"""test for correct key building"""
qb = QueryBuilder(
channel=["test_channel"],
subscribed=["true"],
type=["regular"],
channel="test_channel",
subscribed=True,
type="regular",
)
result = qb.build_data()
must_list = result["query"]["bool"]["must"]
@ -22,7 +22,7 @@ def test_build_data():
def test_parse_type():
"""validate type"""
qb = QueryBuilder(type=["regular"])
qb = QueryBuilder(type="regular")
with pytest.raises(ValueError):
qb.parse_type("invalid")

10
backend/playlist/urls.py
View File

@ -9,6 +9,16 @@ urlpatterns = [
views.PlaylistApiListView.as_view(),
name="api-playlist-list",
),
path(
"custom/",
views.PlaylistCustomApiListView.as_view(),
name="api-custom-playlist-list",
),
path(
"custom/<slug:playlist_id>/",
views.PlaylistCustomApiView.as_view(),
name="api-custom-playlist",
),
path(
"<slug:playlist_id>/",
views.PlaylistApiView.as_view(),

275
backend/playlist/views.py
View File

@ -2,11 +2,25 @@
import uuid
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
)
from common.views_base import AdminWriteOnly, ApiBaseView
from download.src.subscriptions import PlaylistSubscription
from drf_spectacular.utils import OpenApiResponse, extend_schema
from playlist.serializers import (
PlaylistBulkAddSerializer,
PlaylistCustomPostSerializer,
PlaylistDeleteQuerySerializer,
PlaylistListCustomPostSerializer,
PlaylistListQuerySerializer,
PlaylistListSerializer,
PlaylistSerializer,
PlaylistSingleUpdate,
)
from playlist.src.index import YoutubePlaylist
from playlist.src.query_building import QueryBuilder
from rest_framework import status
from rest_framework.response import Response
from task.tasks import subscribe_to
from user.src.user_config import UserConfig
@ -25,58 +39,156 @@ class PlaylistApiListView(ApiBaseView):
search_base = "ta_playlist/_search/"
permission_classes = [AdminWriteOnly]
@extend_schema(
responses={
200: OpenApiResponse(PlaylistListSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
parameters=[PlaylistListQuerySerializer],
)
def get(self, request):
"""get request"""
"""get playlist list"""
query_serializer = PlaylistListQuerySerializer(
data=request.query_params
)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
try:
data = QueryBuilder(**request.GET).build_data()
data = QueryBuilder(**validated_query).build_data()
except ValueError as err:
return Response({"error": str(err)}, status=400)
error = ErrorResponseSerializer({"error": str(err)})
return Response(error.data, status=400)
self.data = data
self.get_document_list(request)
return Response(self.response)
response_serializer = PlaylistListSerializer(self.response)
return Response(response_serializer.data)
@extend_schema(
request=PlaylistBulkAddSerializer(),
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(self, request):
"""subscribe/unsubscribe to list of playlists"""
data = request.data
try:
to_add = data["data"]
except KeyError:
message = "missing expected data key"
print(message)
return Response({"message": message}, status=400)
"""async subscribe to list of playlists"""
data_serializer = PlaylistBulkAddSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
data = data["data"]
if isinstance(data, dict):
custom_name = data.get("create")
if custom_name:
playlist_id = f"TA_playlist_{uuid.uuid4()}"
custom_playlist = YoutubePlaylist(playlist_id)
custom_playlist.create(custom_name)
return Response(custom_playlist.json_data)
pending = [i["playlist_id"] for i in validated_data["data"]]
if not pending:
error = ErrorResponseSerializer({"error": "nothing to subscribe"})
return Response(error.data, status=400)
pending = []
for playlist_item in to_add:
playlist_id = playlist_item["playlist_id"]
if playlist_item["playlist_subscribed"]:
pending.append(playlist_id)
else:
self._unsubscribe(playlist_id)
url_str = " ".join(pending)
task = subscribe_to.delay(url_str, expected_type="playlist")
if pending:
url_str = " ".join(pending)
subscribe_to.delay(url_str, expected_type="playlist")
message = {
"message": "playlist subscribe task started",
"task_id": task.id,
}
serializer = AsyncTaskResponseSerializer(message)
return Response(data)
return Response(serializer.data)
@staticmethod
def _unsubscribe(playlist_id: str):
"""unsubscribe"""
print(f"[{playlist_id}] unsubscribe from playlist")
_ = PlaylistSubscription().change_subscribe(
playlist_id, subscribe_status=False
)
class PlaylistCustomApiListView(ApiBaseView):
"""resolves to /api/playlist/custom/
POST: Create new custom playlist
"""
search_base = "ta_playlist/_search/"
permission_classes = [AdminWriteOnly]
@extend_schema(
request=PlaylistListCustomPostSerializer(),
responses={
200: OpenApiResponse(PlaylistSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(self, request):
"""create new custom playlist"""
serializer = PlaylistListCustomPostSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
custom_name = validated_data["playlist_name"]
playlist_id = f"TA_playlist_{uuid.uuid4()}"
custom_playlist = YoutubePlaylist(playlist_id)
custom_playlist.create(custom_name)
response_serializer = PlaylistSerializer(custom_playlist.json_data)
return Response(response_serializer.data)
class PlaylistCustomApiView(ApiBaseView):
"""resolves to /api/playlist/custom/<playlist_id>/
POST: modify custom playlist
"""
search_base = "ta_playlist/_doc/"
permission_classes = [AdminWriteOnly]
@extend_schema(
request=PlaylistCustomPostSerializer(),
responses={
200: OpenApiResponse(PlaylistSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
404: OpenApiResponse(
ErrorResponseSerializer(), description="playlist not found"
),
},
)
def post(self, request, playlist_id):
"""modify custom playlist"""
data_serializer = PlaylistCustomPostSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
self.get_document(playlist_id)
if not self.response:
error = ErrorResponseSerializer({"error": "playlist not found"})
return Response(error.data, status=404)
if not self.response["playlist_type"] == "custom":
error = ErrorResponseSerializer(
{"error": f"playlist with ID {playlist_id} is not custom"}
)
return Response(error.data, status=400)
action = validated_data.get("action")
video_id = validated_data.get("video_id")
playlist = YoutubePlaylist(playlist_id)
if action == "create":
try:
playlist.add_video_to_playlist(video_id)
except TypeError:
error = ErrorResponseSerializer(
{"error": f"failed to add video {video_id} to playlist"}
)
return Response(error.data, status=400)
else:
hide = UserConfig(request.user.id).get_value("hide_watched")
playlist.move_video(video_id, action, hide_watched=hide)
response_serializer = PlaylistSerializer(playlist.json_data)
return Response(response_serializer.data)
class PlaylistApiView(ApiBaseView):
@ -88,51 +200,74 @@ class PlaylistApiView(ApiBaseView):
permission_classes = [AdminWriteOnly]
valid_custom_actions = ["create", "remove", "up", "down", "top", "bottom"]
@extend_schema(
responses={
200: OpenApiResponse(PlaylistSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="playlist not found"
),
},
)
def get(self, request, playlist_id):
# pylint: disable=unused-argument
"""get request"""
"""get playlist"""
self.get_document(playlist_id)
return Response(self.response, status=self.status_code)
if not self.response:
error = ErrorResponseSerializer({"error": "playlist not found"})
return Response(error.data, status=404)
response_serializer = PlaylistSerializer(self.response)
return Response(response_serializer.data)
@extend_schema(
request=PlaylistSingleUpdate(),
responses={
200: OpenApiResponse(PlaylistSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="playlist not found"
),
},
)
def post(self, request, playlist_id):
"""post to custom playlist to add a video to list"""
"""update subscribed state of playlist"""
data_serializer = PlaylistSingleUpdate(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
self.get_document(playlist_id)
if not self.response["data"]:
return Response({"error": "playlist not found"}, status=404)
if not self.response:
error = ErrorResponseSerializer({"error": "playlist not found"})
return Response(error.data, status=404)
data = request.data
subscribed = data.get("playlist_subscribed")
if subscribed is not None:
playlist_sub = PlaylistSubscription()
json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
return Response(json_data, status=200)
subscribed = validated_data["playlist_subscribed"]
playlist_sub = PlaylistSubscription()
json_data = playlist_sub.change_subscribe(playlist_id, subscribed)
if not self.response["data"]["playlist_type"] == "custom":
message = f"playlist with ID {playlist_id} is not custom"
return Response({"message": message}, status=400)
action = request.data.get("action")
if action not in self.valid_custom_actions:
message = f"invalid action: {action}"
return Response({"message": message}, status=400)
playlist = YoutubePlaylist(playlist_id)
video_id = request.data.get("video_id")
if action == "create":
playlist.add_video_to_playlist(video_id)
else:
hide = UserConfig(request.user.id).get_value("hide_watched")
playlist.move_video(video_id, action, hide_watched=hide)
return Response({"success": True}, status=status.HTTP_201_CREATED)
response_serializer = PlaylistSerializer(json_data)
return Response(response_serializer.data)
@extend_schema(
parameters=[PlaylistDeleteQuerySerializer],
responses={
204: OpenApiResponse(description="playlist deleted"),
},
)
def delete(self, request, playlist_id):
"""delete playlist"""
print(f"{playlist_id}: delete playlist")
delete_videos = request.GET.get("delete-videos", False)
query_serializer = PlaylistDeleteQuerySerializer(
data=request.query_params
)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
delete_videos = validated_query.get("delete_videos", False)
if delete_videos:
YoutubePlaylist(playlist_id).delete_videos_playlist()
else:
YoutubePlaylist(playlist_id).delete_metadata()
return Response({"success": True})
return Response(status=204)

4
backend/requirements-dev.txt
View File

@ -1,8 +1,8 @@
-r requirements.txt
ipython==8.31.0
ipython==8.32.0
pre-commit==4.1.0
pylint-django==2.6.1
pylint==3.3.3
pylint==3.3.4
pytest-django==4.9.0
pytest==8.3.4
python-dotenv==1.0.1

7
backend/requirements.txt
View File

@ -2,13 +2,14 @@ apprise==1.9.2
celery==5.4.0
django-auth-ldap==5.1.0
django-celery-beat==2.7.0
django-cors-headers==4.6.0
Django==5.1.5
django-cors-headers==4.7.0
Django==5.1.6
djangorestframework==3.15.2
drf-spectacular==0.28.0
Pillow==11.1.0
redis==5.2.1
requests==2.32.3
ryd-client==0.0.6
uvicorn==0.34.0
whitenoise==6.8.2
whitenoise==6.9.0
yt-dlp[default]==2025.1.26

110
backend/stats/serializers.py Normal file
View File

@ -0,0 +1,110 @@
"""serializers for stats"""
# pylint: disable=abstract-method
from rest_framework import serializers
class VideoStatsItemSerializer(serializers.Serializer):
"""serialize video stats item"""
doc_count = serializers.IntegerField()
media_size = serializers.IntegerField()
duration = serializers.IntegerField()
duration_str = serializers.CharField()
class VideoStatsSerializer(serializers.Serializer):
"""serialize video stats"""
doc_count = serializers.IntegerField()
media_size = serializers.IntegerField()
duration = serializers.IntegerField()
duration_str = serializers.CharField()
type_videos = VideoStatsItemSerializer(allow_null=True)
type_shorts = VideoStatsItemSerializer(allow_null=True)
type_streams = VideoStatsItemSerializer(allow_null=True)
active_true = VideoStatsItemSerializer(allow_null=True)
active_false = VideoStatsItemSerializer(allow_null=True)
class ChannelStatsSerializer(serializers.Serializer):
"""serialize channel stats"""
doc_count = serializers.IntegerField(allow_null=True)
active_true = serializers.IntegerField(allow_null=True)
active_false = serializers.IntegerField(allow_null=True)
subscribed_true = serializers.IntegerField(allow_null=True)
subscribed_false = serializers.IntegerField(allow_null=True)
class PlaylistStatsSerializer(serializers.Serializer):
"""serialize playlists stats"""
doc_count = serializers.IntegerField(allow_null=True)
active_true = serializers.IntegerField(allow_null=True)
active_false = serializers.IntegerField(allow_null=True)
subscribed_false = serializers.IntegerField(allow_null=True)
subscribed_true = serializers.IntegerField(allow_null=True)
class DownloadStatsSerializer(serializers.Serializer):
"""serialize download stats"""
pending = serializers.IntegerField(allow_null=True)
ignore = serializers.IntegerField(allow_null=True)
pending_videos = serializers.IntegerField(allow_null=True)
pending_shorts = serializers.IntegerField(allow_null=True)
pending_streams = serializers.IntegerField(allow_null=True)
class WatchTotalStatsSerializer(serializers.Serializer):
"""serialize total watch stats"""
duration = serializers.IntegerField()
duration_str = serializers.CharField()
items = serializers.IntegerField()
class WatchItemStatsSerializer(serializers.Serializer):
"""serialize watch item stats"""
duration = serializers.IntegerField()
duration_str = serializers.CharField()
progress = serializers.FloatField()
items = serializers.IntegerField()
class WatchStatsSerializer(serializers.Serializer):
"""serialize watch stats"""
total = WatchTotalStatsSerializer(allow_null=True)
unwatched = WatchItemStatsSerializer(allow_null=True)
watched = WatchItemStatsSerializer(allow_null=True)
class DownloadHistItemSerializer(serializers.Serializer):
"""serialize download hist item"""
date = serializers.CharField()
count = serializers.IntegerField()
media_size = serializers.IntegerField()
class BiggestChannelQuerySerializer(serializers.Serializer):
"""serialize biggest channel query"""
order = serializers.ChoiceField(
choices=["doc_count", "duration", "media_size"], default="doc_count"
)
class BiggestChannelItemSerializer(serializers.Serializer):
"""serialize biggest channel item"""
id = serializers.CharField()
name = serializers.CharField()
doc_count = serializers.IntegerField()
duration = serializers.IntegerField()
duration_str = serializers.CharField()
media_size = serializers.IntegerField()

77
backend/stats/views.py
View File

@ -1,7 +1,19 @@
"""all stats API views"""
from common.serializers import ErrorResponseSerializer
from common.views_base import ApiBaseView
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.response import Response
from stats.serializers import (
BiggestChannelItemSerializer,
BiggestChannelQuerySerializer,
ChannelStatsSerializer,
DownloadHistItemSerializer,
DownloadStatsSerializer,
PlaylistStatsSerializer,
VideoStatsSerializer,
WatchStatsSerializer,
)
from stats.src.aggs import (
BiggestChannel,
Channel,
@ -18,11 +30,13 @@ class StatVideoView(ApiBaseView):
GET: return video stats
"""
@extend_schema(responses=VideoStatsSerializer())
def get(self, request):
"""get stats"""
"""get video stats"""
# pylint: disable=unused-argument
serializer = VideoStatsSerializer(Video().process())
return Response(Video().process())
return Response(serializer.data)
class StatChannelView(ApiBaseView):
@ -30,11 +44,13 @@ class StatChannelView(ApiBaseView):
GET: return channel stats
"""
@extend_schema(responses=ChannelStatsSerializer())
def get(self, request):
"""get stats"""
"""get channel stats"""
# pylint: disable=unused-argument
serializer = ChannelStatsSerializer(Channel().process())
return Response(Channel().process())
return Response(serializer.data)
class StatPlaylistView(ApiBaseView):
@ -42,11 +58,13 @@ class StatPlaylistView(ApiBaseView):
GET: return playlist stats
"""
@extend_schema(responses=PlaylistStatsSerializer())
def get(self, request):
"""get stats"""
"""get playlist stats"""
# pylint: disable=unused-argument
serializer = PlaylistStatsSerializer(Playlist().process())
return Response(Playlist().process())
return Response(serializer.data)
class StatDownloadView(ApiBaseView):
@ -54,23 +72,27 @@ class StatDownloadView(ApiBaseView):
GET: return download stats
"""
@extend_schema(responses=DownloadStatsSerializer())
def get(self, request):
"""get stats"""
"""get download stats"""
# pylint: disable=unused-argument
serializer = DownloadStatsSerializer(Download().process())
return Response(Download().process())
return Response(serializer.data)
class StatWatchProgress(ApiBaseView):
"""resolves to /api/stats/watchprogress/
"""resolves to /api/stats/watch/
GET: return watch/unwatch progress stats
"""
@extend_schema(responses=WatchStatsSerializer())
def get(self, request):
"""handle get request"""
"""get watched stats"""
# pylint: disable=unused-argument
serializer = WatchStatsSerializer(WatchProgress().process())
return Response(WatchProgress().process())
return Response(serializer.data)
class StatDownloadHist(ApiBaseView):
@ -78,11 +100,14 @@ class StatDownloadHist(ApiBaseView):
GET: return download video count histogram for last days
"""
@extend_schema(responses=DownloadHistItemSerializer(many=True))
def get(self, request):
"""handle get request"""
"""get download hist items"""
# pylint: disable=unused-argument
download_items = DownloadHist().process()
serializer = DownloadHistItemSerializer(download_items, many=True)
return Response(DownloadHist().process())
return Response(serializer.data)
class StatBiggestChannel(ApiBaseView):
@ -91,14 +116,24 @@ class StatBiggestChannel(ApiBaseView):
param: order
"""
order_choices = ["doc_count", "duration", "media_size"]
@extend_schema(
responses={
200: OpenApiResponse(BiggestChannelItemSerializer(many=True)),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def get(self, request):
"""handle get request"""
"""get biggest channels stats"""
query_serializer = BiggestChannelQuerySerializer(
data=request.query_params
)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
order = validated_query["order"]
order = request.GET.get("order", "doc_count")
if order and order not in self.order_choices:
message = {"message": f"invalid order parameter {order}"}
return Response(message, status=400)
channel_items = BiggestChannel(order).process()
serializer = BiggestChannelItemSerializer(channel_items, many=True)
return Response(BiggestChannel(order).process())
return Response(serializer.data)

70
backend/task/serializers.py
View File

@ -1,7 +1,10 @@
"""serializer for tasks"""
# pylint: disable=abstract-method
from rest_framework import serializers
from task.models import CustomPeriodicTask
from task.src.task_config import TASK_CONFIG
class CustomPeriodicTaskSerializer(serializers.ModelSerializer):
@ -21,3 +24,70 @@ class CustomPeriodicTaskSerializer(serializers.ModelSerializer):
"last_run_at",
"config",
]
class TaskResultSerializer(serializers.Serializer):
"""serialize task result stored in redis"""
status = serializers.ChoiceField(
choices=[
"PENDING",
"STARTED",
"SUCCESS",
"FAILURE",
"RETRY",
"REVOKED",
]
)
result = serializers.CharField(allow_null=True)
traceback = serializers.CharField(allow_null=True)
date_done = serializers.CharField()
name = serializers.CharField()
args = serializers.ListField(child=serializers.JSONField(), required=False)
children = serializers.ListField(
child=serializers.CharField(), required=False
)
kwargs = serializers.DictField(required=False)
worker = serializers.CharField(required=False)
retries = serializers.IntegerField(required=False)
queue = serializers.CharField(required=False)
task_id = serializers.CharField()
class TaskIDDataSerializer(serializers.Serializer):
"""serialize task by ID POST data"""
command = serializers.ChoiceField(choices=["stop", "kill"])
class TaskCreateDataSerializer(serializers.Serializer):
"""serialize task create data"""
schedule = serializers.CharField(required=False)
config = serializers.DictField(required=False)
class TaskNotificationItemSerializer(serializers.Serializer):
"""serialize single task notification"""
urls = serializers.ListField(child=serializers.CharField())
title = serializers.CharField()
def create_dynamic_notification_serializer():
"""use task config"""
fields = {
key: TaskNotificationItemSerializer(required=False)
for key in TASK_CONFIG
}
return type("DynamicDictSerializer", (serializers.Serializer,), fields)
TaskNotificationSerializer = create_dynamic_notification_serializer()
class TaskNotificationPostSerializer(serializers.Serializer):
"""serialize task notification POST"""
task_name = serializers.ChoiceField(choices=list(TASK_CONFIG))
url = serializers.CharField(required=False)

22
backend/task/src/config_schedule.py
View File

@ -28,18 +28,19 @@ class ScheduleBuilder:
def update_schedule(
self, task_name: str, cron_schedule: str, schedule_conf: dict | None
) -> None:
) -> CustomPeriodicTask:
"""update schedule"""
if cron_schedule == "auto":
cron_schedule = self.SCHEDULES[task_name]
if cron_schedule:
_ = self.get_set_task(task_name, cron_schedule)
task = self.get_set_task(task_name, cron_schedule)
if schedule_conf:
for key, value in schedule_conf.items():
self.set_config(task_name, key, value)
return task
def get_set_task(self, task_name, schedule=False):
"""get task"""
try:
@ -69,14 +70,15 @@ class ScheduleBuilder:
return task_crontab
def set_config(self, task_name: str, key: str, value) -> None:
def set_config(
self, task_name: str, key: str, value
) -> CustomPeriodicTask:
"""set task_config, validate before"""
try:
task = CustomPeriodicTask.objects.get(name=task_name)
task.task_config.update({key: value})
task.save()
except CustomPeriodicTask.DoesNotExist:
pass
task = CustomPeriodicTask.objects.get(name=task_name)
task.task_config.update({key: value})
task.save()
return task
class CrontabValidator:

254
backend/task/views.py
View File

@ -1,10 +1,22 @@
"""all task API views"""
from common.serializers import (
AsyncTaskResponseSerializer,
ErrorResponseSerializer,
)
from common.views_base import AdminOnly, ApiBaseView
from django.shortcuts import get_object_or_404
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.response import Response
from task.models import CustomPeriodicTask
from task.serializers import CustomPeriodicTaskSerializer
from task.serializers import (
CustomPeriodicTaskSerializer,
TaskCreateDataSerializer,
TaskIDDataSerializer,
TaskNotificationPostSerializer,
TaskNotificationSerializer,
TaskResultSerializer,
)
from task.src.config_schedule import CrontabValidator, ScheduleBuilder
from task.src.notify import Notifications, get_all_notifications
from task.src.task_config import TASK_CONFIG
@ -18,12 +30,14 @@ class TaskListView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(responses=TaskResultSerializer(many=True))
def get(self, request):
"""handle get request"""
"""get all stored task results"""
# pylint: disable=unused-argument
all_results = TaskManager().get_all_results()
serializer = TaskResultSerializer(all_results, many=True)
return Response(all_results)
return Response(serializer.data)
class TaskNameListView(ApiBaseView):
@ -34,36 +48,55 @@ class TaskNameListView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(TaskResultSerializer(many=True)),
404: OpenApiResponse(
ErrorResponseSerializer(), description="task name not found"
),
},
)
def get(self, request, task_name):
"""handle get request"""
"""get stored task by name"""
# pylint: disable=unused-argument
if task_name not in TASK_CONFIG:
message = {"message": "invalid task name"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "task name not found"})
return Response(error.data, status=404)
all_results = TaskManager().get_tasks_by_name(task_name)
serializer = TaskResultSerializer(all_results, many=True)
return Response(all_results)
return Response(serializer.data)
@extend_schema(
responses={
200: OpenApiResponse(AsyncTaskResponseSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
404: OpenApiResponse(
ErrorResponseSerializer(), description="task name not found"
),
}
)
def post(self, request, task_name):
"""
handle post request
404 for invalid task_name
400 if task can't be started here without argument
"""
"""start new task without args"""
# pylint: disable=unused-argument
task_config = TASK_CONFIG.get(task_name)
if not task_config:
message = {"message": "invalid task name"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "task name not found"})
return Response(error.data, status=404)
if not task_config.get("api_start"):
message = {"message": "can not start task through this endpoint"}
return Response(message, status=400)
error = ErrorResponseSerializer(
{"error": "can not start task through this endpoint"}
)
return Response(error.data, status=404)
message = TaskCommand().start(task_name)
serializer = AsyncTaskResponseSerializer(message)
return Response({"message": message})
return Response(serializer.data)
class TaskIDView(ApiBaseView):
@ -75,43 +108,70 @@ class TaskIDView(ApiBaseView):
valid_commands = ["stop", "kill"]
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(TaskResultSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="task not found"
),
},
)
def get(self, request, task_id):
"""handle get request"""
"""get task by ID"""
# pylint: disable=unused-argument
task_result = TaskManager().get_task(task_id)
if not task_result:
message = {"message": "task id not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "task ID not found"})
return Response(error.data, status=404)
return Response(task_result)
serializer = TaskResultSerializer(task_result)
return Response(serializer.data)
@extend_schema(
request=TaskIDDataSerializer(),
responses={
204: OpenApiResponse(description="task command sent"),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
404: OpenApiResponse(
ErrorResponseSerializer(), description="task not found"
),
},
)
def post(self, request, task_id):
"""post command to task"""
command = request.data.get("command")
if not command or command not in self.valid_commands:
message = {"message": "no valid command found"}
return Response(message, status=400)
data_serializer = TaskIDDataSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
command = validated_data["command"]
task_result = TaskManager().get_task(task_id)
if not task_result:
message = {"message": "task id not found"}
return Response(message, status=404)
error = ErrorResponseSerializer({"error": "task ID not found"})
return Response(error.data, status=404)
task_conf = TASK_CONFIG.get(task_result.get("name"))
if command == "stop":
if not task_conf.get("api_stop"):
message = {"message": "task can not be stopped"}
return Response(message, status=400)
error = ErrorResponseSerializer(
{"error": "task can not be stopped"}
)
return Response(error.data, status=400)
TaskCommand().stop(task_id)
if command == "kill":
if not task_conf.get("api_stop"):
message = {"message": "task can not be killed"}
return Response(message, status=400)
error = ErrorResponseSerializer(
{"error": "task can not be killed"}
)
return Response(error.data, status=400)
TaskCommand().kill(task_id)
return Response({"message": "command sent"})
return Response(status=204)
class ScheduleListView(ApiBaseView):
@ -121,11 +181,16 @@ class ScheduleListView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(CustomPeriodicTaskSerializer(many=True)),
},
)
def get(self, request):
"""get all schedules"""
tasks = CustomPeriodicTask.objects.all()
response = CustomPeriodicTaskSerializer(tasks, many=True).data
return Response(response)
serializer = CustomPeriodicTaskSerializer(tasks, many=True)
return Response(serializer.data)
class ScheduleView(ApiBaseView):
@ -137,42 +202,77 @@ class ScheduleView(ApiBaseView):
permission_classes = [AdminOnly]
@extend_schema(
responses={
200: OpenApiResponse(CustomPeriodicTaskSerializer()),
404: OpenApiResponse(
ErrorResponseSerializer(), description="schedule not found"
),
},
)
def get(self, request, task_name):
"""get single schedule by task_name"""
task = get_object_or_404(CustomPeriodicTask, name=task_name)
response = CustomPeriodicTaskSerializer(task).data
return Response(response)
serializer = CustomPeriodicTaskSerializer(task)
return Response(serializer.data)
@extend_schema(
request=TaskCreateDataSerializer(),
responses={
200: OpenApiResponse(CustomPeriodicTaskSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
},
)
def post(self, request, task_name):
"""create/update schedule for task"""
cron_schedule = request.data.get("schedule")
schedule_config = request.data.get("config")
data_serializer = TaskCreateDataSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
cron_schedule = validated_data.get("schedule")
schedule_config = validated_data.get("config")
if not cron_schedule and not schedule_config:
message = {"message": "expected schedule or config key"}
return Response(message, status=400)
error = ErrorResponseSerializer(
{"error": "expected schedule or config key"}
)
return Response(error.data, status=400)
try:
validator = CrontabValidator()
validator.validate_cron(cron_schedule)
validator.validate_config(task_name, schedule_config)
except ValueError as err:
return Response({"message": str(err)}, status=400)
error = ErrorResponseSerializer({"error": str(err)})
return Response(error.data, status=400)
ScheduleBuilder().update_schedule(
task = ScheduleBuilder().update_schedule(
task_name, cron_schedule, schedule_config
)
message = f"update schedule for task {task_name}"
if schedule_config:
message += f" with config {schedule_config}"
return Response({"message": message})
print(message)
serializer = CustomPeriodicTaskSerializer(task)
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="schedule deleted"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="schedule not found"
),
},
)
def delete(self, request, task_name):
"""delete schedule by task_name query"""
"""delete schedule by task_name"""
task = get_object_or_404(CustomPeriodicTask, name=task_name)
_ = task.delete()
return Response({"success": True})
return Response(status=204)
class ScheduleNotification(ApiBaseView):
@ -182,42 +282,64 @@ class ScheduleNotification(ApiBaseView):
DEL: delete notification
"""
@extend_schema(
responses=TaskNotificationSerializer(),
)
def get(self, request):
"""handle get request"""
serializer = TaskNotificationSerializer(get_all_notifications())
return Response(get_all_notifications())
return Response(serializer.data)
@extend_schema(
request=TaskNotificationPostSerializer(),
responses={
200: OpenApiResponse(TaskNotificationSerializer()),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
},
)
def post(self, request):
"""handle create notification"""
task_name = request.data.get("task_name")
url = request.data.get("url")
if not TASK_CONFIG.get(task_name):
message = {"message": "task_name not found"}
return Response(message, status=404)
"""create notification"""
data_serializer = TaskNotificationPostSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
task_name = validated_data["task_name"]
url = validated_data["url"]
if not url:
message = {"message": "missing url key"}
return Response(message, status=400)
error = ErrorResponseSerializer({"error": "missing url"})
return Response(error.data, status=400)
Notifications(task_name).add_url(url)
message = {"task_name": task_name, "url": url}
return Response(message)
serializer = TaskNotificationSerializer(get_all_notifications())
return Response(serializer.data)
@extend_schema(
request=TaskNotificationPostSerializer(),
responses={
204: OpenApiResponse(description="notification url deleted"),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
},
)
def delete(self, request):
"""handle delete"""
"""delete notification"""
task_name = request.data.get("task_name")
url = request.data.get("url")
data_serializer = TaskNotificationPostSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
if not TASK_CONFIG.get(task_name):
message = {"message": "task_name not found"}
return Response(message, status=404)
task_name = validated_data["task_name"]
url = validated_data.get("url")
if url:
response, status_code = Notifications(task_name).remove_url(url)
Notifications(task_name).remove_url(url)
else:
response, status_code = Notifications(task_name).remove_task()
Notifications(task_name).remove_task()
return Response({"response": response, "status_code": status_code})
return Response(status=204)

30
backend/user/serializers.py
View File

@ -1,7 +1,11 @@
"""serializer for account model"""
# pylint: disable=abstract-method
from common.src.helper import get_stylesheets
from rest_framework import serializers
from user.models import Account
from video.src.constants import OrderEnum, SortEnum
class AccountSerializer(serializers.ModelSerializer):
@ -18,3 +22,29 @@ class AccountSerializer(serializers.ModelSerializer):
"user_permissions",
"last_login",
)
class UserMeConfigSerializer(serializers.Serializer):
"""serialize user me config"""
stylesheet = serializers.ChoiceField(choices=get_stylesheets())
page_size = serializers.IntegerField()
sort_by = serializers.ChoiceField(choices=SortEnum.names())
sort_order = serializers.ChoiceField(choices=OrderEnum.values())
view_style_home = serializers.ChoiceField(choices=["grid", "list"])
view_style_channel = serializers.ChoiceField(choices=["grid", "list"])
view_style_downloads = serializers.ChoiceField(choices=["grid", "list"])
view_style_playlist = serializers.ChoiceField(choices=["grid", "list"])
grid_items = serializers.IntegerField(max_value=7, min_value=3)
hide_watched = serializers.BooleanField()
show_ignored_only = serializers.BooleanField()
show_subed_only = serializers.BooleanField()
show_help_text = serializers.BooleanField()
class LoginSerializer(serializers.Serializer):
"""serialize login"""
username = serializers.CharField()
password = serializers.CharField()
remember_me = serializers.ChoiceField(choices=["on", "off"], default="off")

67
backend/user/src/user_config.py
View File

@ -7,7 +7,6 @@ Functionality:
from typing import TypedDict
from common.src.es_connect import ElasticWrap
from common.src.helper import get_stylesheets
class UserConfigType(TypedDict, total=False):
@ -29,7 +28,10 @@ class UserConfigType(TypedDict, total=False):
class UserConfig:
"""Handle settings for an individual user"""
"""
Handle settings for an individual user
items are expected to be validated in the serializer
"""
_DEFAULT_USER_SETTINGS = UserConfigType(
stylesheet="dark.css",
@ -47,19 +49,6 @@ class UserConfig:
show_help_text=True,
)
VALID_STYLESHEETS = get_stylesheets()
VALID_VIEW_STYLE = ["grid", "list"]
VALID_SORT_ORDER = ["asc", "desc"]
VALID_SORT_BY = [
"published",
"downloaded",
"views",
"likes",
"duration",
"filesize",
]
VALID_GRID_ITEMS = range(3, 8)
def __init__(self, user_id: str):
self._user_id: str = user_id
self._config: UserConfigType = self.get_config()
@ -84,7 +73,6 @@ class UserConfig:
def set_value(self, key: str, value: str | bool | int):
"""Set or replace a configuration value for the user"""
self._validate(key, value)
data = {"doc": {"config": {key: value}}}
response, status = ElasticWrap(self.es_update_url).post(data)
if status < 200 or status > 299:
@ -92,43 +80,6 @@ class UserConfig:
print(f"User {self._user_id} value '{key}' change: to {value}")
def _validate(self, key, value):
"""validate key and value"""
if not self._user_id:
raise ValueError("Unable to persist config for null user_id")
if key not in self._DEFAULT_USER_SETTINGS:
raise KeyError(
f"Unable to persist config for an unknown key '{key}'"
)
valid_values = {
"stylesheet": self.VALID_STYLESHEETS,
"sort_by": self.VALID_SORT_BY,
"sort_order": self.VALID_SORT_ORDER,
"view_style_home": self.VALID_VIEW_STYLE,
"view_style_channel": self.VALID_VIEW_STYLE,
"view_style_download": self.VALID_VIEW_STYLE,
"view_style_playlist": self.VALID_VIEW_STYLE,
"grid_items": self.VALID_GRID_ITEMS,
"page_size": int,
"hide_watched": bool,
"show_ignored_only": bool,
"show_subed_only": bool,
"show_help_text": bool,
}
validation_value = valid_values.get(key)
if isinstance(validation_value, (list, range)):
if value not in validation_value:
raise ValueError(f"Invalid value for {key}: {value}")
elif validation_value == int:
if not isinstance(value, int):
raise ValueError(f"Invalid value for {key}: {value}")
elif validation_value == bool:
if not isinstance(value, bool):
raise ValueError(f"Invalid value for {key}: {value}")
def get_config(self) -> UserConfigType:
"""get config from ES or load from the application defaults"""
if not self._user_id:
@ -143,6 +94,16 @@ class UserConfig:
return config
def update_config(self, to_update: dict) -> None:
"""update config object"""
data = {"doc": {"config": to_update}}
response, status = ElasticWrap(self.es_update_url).post(data)
if status < 200 or status > 299:
raise ValueError(f"Failed storing user value {status}: {response}")
for key, value in to_update.items():
print(f"User {self._user_id} value '{key}' change: to {value}")
def sync_defaults(self):
"""set initial defaults on 404"""
response, _ = ElasticWrap(self.es_url).post(

1
backend/user/urls.py
View File

@ -6,5 +6,6 @@ from user import views
urlpatterns = [
path("login/", views.LoginApiView.as_view(), name="api-user-login"),
path("logout/", views.LogoutApiView.as_view(), name="api-user-logout"),
path("account/", views.UserAccountView.as_view(), name="api-user-account"),
path("me/", views.UserConfigView.as_view(), name="api-user-me"),
]

119
backend/user/views.py
View File

@ -1,63 +1,75 @@
"""all user api views"""
from common.serializers import ErrorResponseSerializer
from common.views import ApiBaseView
from django.contrib.auth import authenticate, login, logout
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from drf_spectacular.utils import OpenApiResponse, extend_schema
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from user.models import Account
from user.serializers import AccountSerializer
from user.serializers import (
AccountSerializer,
LoginSerializer,
UserMeConfigSerializer,
)
from user.src.user_config import UserConfig
class UserAccountView(ApiBaseView):
"""resolves to /api/user/account/
GET: return current user account
"""
@extend_schema(
responses=AccountSerializer(),
)
def get(self, request):
"""get user account"""
user_id = request.user.id
account = Account.objects.get(id=user_id)
account_serializer = AccountSerializer(account)
return Response(account_serializer.data)
class UserConfigView(ApiBaseView):
"""resolves to /api/user/me/
GET: return current user config
POST: update user config
"""
@extend_schema(responses=UserMeConfigSerializer())
def get(self, request):
"""get config"""
user_id = request.user.id
account = Account.objects.get(id=user_id)
serializer = AccountSerializer(account)
response = serializer.data.copy()
"""get user config"""
config = UserConfig(request.user.id).get_config()
serializer = UserMeConfigSerializer(config)
config = UserConfig(user_id).get_config()
response.update({"config": config})
return Response(response)
return Response(serializer.data)
@extend_schema(
request=UserMeConfigSerializer(required=False),
responses={
200: UserMeConfigSerializer(),
400: OpenApiResponse(
ErrorResponseSerializer(), description="Bad request"
),
},
)
def post(self, request):
"""update config"""
user_id = request.user.id
data = request.data
"""update config, allows partial update"""
data_config = data.get("config")
if not data_config:
message = {
"status": "Bad Request",
"message": "missing config key",
}
return Response(message, status=400)
data_serializer = UserMeConfigSerializer(
data=request.data, partial=True
)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
UserConfig(request.user.id).update_config(to_update=validated_data)
config = UserConfig(request.user.id).get_config()
serializer = UserMeConfigSerializer(config)
user_conf = UserConfig(user_id)
for key, value in data_config.items():
try:
user_conf.set_value(key, value)
except ValueError as err:
message = {
"status": "Bad Request",
"message": f"failed updating {key} to '{value}', {err}",
}
return Response(message, status=400)
response = user_conf.get_config()
response.update({"user_id": user_id})
return Response(response)
return Response(serializer.data)
@method_decorator(csrf_exempt, name="dispatch")
@ -67,21 +79,37 @@ class LoginApiView(APIView):
"""
permission_classes = [AllowAny]
SEC_IN_DAY = 60 * 60 * 24
@extend_schema(
request=LoginSerializer(),
responses={204: OpenApiResponse(description="login successful")},
)
def post(self, request, *args, **kwargs):
"""post data"""
"""login with username and password"""
# pylint: disable=no-member
data_serializer = LoginSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
username = request.data.get("username")
password = request.data.get("password")
username = validated_data["username"]
password = validated_data["password"]
remember_me = validated_data.get("remember_me")
user = authenticate(request, username=username, password=password)
if user is None:
error = ErrorResponseSerializer({"error": "Invalid credentials"})
return Response(error.data, status=400)
if user is not None:
login(request, user) # Creates a session for the user
return Response({"message": "Login successful"}, status=200)
if remember_me == "on":
request.session.set_expiry(self.SEC_IN_DAY * 365)
else:
request.session.set_expiry(self.SEC_IN_DAY * 2)
return Response({"message": "Invalid credentials"}, status=400)
print(f"expire session in {request.session.get_expiry_age()} secs")
login(request, user) # Creates a session for the user
return Response(status=204)
class LogoutApiView(ApiBaseView):
@ -89,7 +117,10 @@ class LogoutApiView(ApiBaseView):
POST: handle logout
"""
@extend_schema(
responses={204: OpenApiResponse(description="logout successful")}
)
def post(self, request, *args, **kwargs):
"""logout on post request"""
"""logout user from session"""
logout(request)
return Response({"message": "Successfully logged out."}, status=200)
return Response(status=204)

187
backend/video/serializers.py Normal file
View File

@ -0,0 +1,187 @@
"""video serializers"""
# pylint: disable=abstract-method
from channel.serializers import ChannelSerializer
from common.serializers import PaginationSerializer
from rest_framework import serializers
from video.src.constants import OrderEnum, SortEnum, VideoTypeEnum, WatchedEnum
class PlayerSerializer(serializers.Serializer):
"""serialize player"""
watched = serializers.BooleanField()
duration = serializers.IntegerField()
duration_str = serializers.CharField()
progress = serializers.FloatField(required=False)
position = serializers.FloatField(required=False)
class SponsorBlockSegmentSerializer(serializers.Serializer):
"""serialize sponsorblock segment"""
actionType = serializers.CharField()
videoDuration = serializers.FloatField()
segment = serializers.ListField(child=serializers.FloatField())
votes = serializers.IntegerField()
category = serializers.CharField()
UUID = serializers.CharField()
locked = serializers.IntegerField()
class SponsorBlockSerializer(serializers.Serializer):
"""serialize sponsorblock"""
is_enabled = serializers.BooleanField()
last_refresh = serializers.IntegerField()
has_unlocked = serializers.BooleanField(required=False)
segments = SponsorBlockSegmentSerializer(many=True)
class StatsSerializer(serializers.Serializer):
"""serialize stats"""
like_count = serializers.IntegerField(required=False)
average_rating = serializers.FloatField(required=False)
view_count = serializers.IntegerField(required=False)
dislike_count = serializers.IntegerField(required=False)
class StreamItemSerializer(serializers.Serializer):
"""serialize stream item"""
index = serializers.IntegerField()
codec = serializers.CharField()
bitrate = serializers.IntegerField()
type = serializers.ChoiceField(choices=["video", "audio"])
width = serializers.IntegerField(required=False)
height = serializers.IntegerField(required=False)
class SubtitleItemSerializer(serializers.Serializer):
"""serialize subtitle item"""
ext = serializers.ChoiceField(choices=["json3"])
name = serializers.CharField()
source = serializers.ChoiceField(choices=["user", "auto"])
lang = serializers.CharField()
media_url = serializers.CharField()
url = serializers.URLField()
class VideoSerializer(serializers.Serializer):
"""serialize video item"""
active = serializers.BooleanField()
category = serializers.ListField(child=serializers.CharField())
channel = ChannelSerializer()
comment_count = serializers.IntegerField(allow_null=True)
date_downloaded = serializers.IntegerField()
description = serializers.CharField()
media_size = serializers.IntegerField()
media_url = serializers.CharField()
player = PlayerSerializer()
playlist = serializers.ListField(child=serializers.CharField())
published = serializers.CharField()
sponsorblock = SponsorBlockSerializer(allow_null=True)
stats = StatsSerializer()
streams = StreamItemSerializer(many=True)
subtitles = SubtitleItemSerializer(many=True)
tags = serializers.ListField(child=serializers.CharField())
title = serializers.CharField()
vid_last_refresh = serializers.CharField()
vid_thumb_url = serializers.CharField()
vid_type = serializers.ChoiceField(choices=VideoTypeEnum.values_known())
youtube_id = serializers.CharField()
_index = serializers.CharField(required=False)
_score = serializers.FloatField(required=False)
class VideoListSerializer(serializers.Serializer):
"""serialize video list"""
data = VideoSerializer(many=True)
paginate = PaginationSerializer()
class VideoListQuerySerializer(serializers.Serializer):
"""serialize query for video list"""
playlist = serializers.CharField(required=False)
channel = serializers.CharField(required=False)
watch = serializers.ChoiceField(
choices=WatchedEnum.values(), required=False
)
sort = serializers.ChoiceField(choices=SortEnum.names(), required=False)
order = serializers.ChoiceField(choices=OrderEnum.values(), required=False)
type = serializers.ChoiceField(
choices=VideoTypeEnum.values_known(), required=False
)
class CommentThreadItemSerializer(serializers.Serializer):
"""serialize comment thread item"""
comment_id = serializers.CharField()
comment_text = serializers.CharField()
comment_timestamp = serializers.IntegerField()
comment_time_text = serializers.CharField()
comment_likecount = serializers.IntegerField()
comment_is_favorited = serializers.BooleanField()
comment_author = serializers.CharField()
comment_author_id = serializers.CharField()
comment_author_thumbnail = serializers.URLField()
comment_author_is_uploader = serializers.BooleanField()
comment_parent = serializers.CharField()
class CommentItemSerializer(serializers.Serializer):
"""serialize comment item"""
comment_id = serializers.CharField()
comment_text = serializers.CharField()
comment_timestamp = serializers.IntegerField()
comment_time_text = serializers.CharField()
comment_likecount = serializers.IntegerField()
comment_is_favorited = serializers.BooleanField()
comment_author = serializers.CharField()
comment_author_id = serializers.CharField()
comment_author_thumbnail = serializers.URLField()
comment_author_is_uploader = serializers.BooleanField()
comment_parent = serializers.CharField()
comment_replies = CommentThreadItemSerializer(many=True)
class PlaylistNavMetaSerializer(serializers.Serializer):
"""serialize playlist nav meta"""
current_idx = serializers.IntegerField()
playlist_id = serializers.CharField()
playlist_name = serializers.CharField()
playlist_channel = serializers.CharField()
class PlaylistNavVideoSerializer(serializers.Serializer):
"""serialize video item on playlist nav"""
youtube_id = serializers.CharField()
title = serializers.CharField()
uploader = serializers.CharField()
idx = serializers.IntegerField()
downloaded = serializers.BooleanField()
vid_thumb = serializers.CharField()
class PlaylistNavItemSerializer(serializers.Serializer):
"""serialize nav on playlist"""
playlist_meta = PlaylistNavMetaSerializer()
playlist_previous = PlaylistNavVideoSerializer(allow_null=True)
playlist_next = PlaylistNavVideoSerializer(allow_null=True)
class VideoProgressUpdateSerializer(serializers.Serializer):
"""serialize progress update data"""
position = serializers.FloatField(default=0)

38
backend/video/src/constants.py
View File

@ -11,6 +11,16 @@ class VideoTypeEnum(enum.Enum):
SHORTS = "shorts"
UNKNOWN = "unknown"
@classmethod
def values(cls) -> list[str]:
"""value list"""
return [i.value for i in cls]
@classmethod
def values_known(cls) -> list[str]:
"""values known"""
return [i.value for i in cls if i.value != "unknown"]
class SortEnum(enum.Enum):
"""all sort by options"""
@ -22,9 +32,37 @@ class SortEnum(enum.Enum):
DURATION = "player.duration"
MEDIASIZE = "media_size"
@classmethod
def values(cls) -> list[str]:
"""value list"""
return [i.value for i in cls]
@classmethod
def names(cls) -> list[str]:
"""name list"""
return [i.name.lower() for i in cls]
class OrderEnum(enum.Enum):
"""all order by options"""
ASC = "asc"
DESC = "desc"
@classmethod
def values(cls) -> list[str]:
"""value list"""
return [i.value for i in cls]
class WatchedEnum(enum.Enum):
"""watched state enum"""
WATCHED = "watched"
UNWATCHED = "unwatched"
CONTINUE = "continue"
@classmethod
def values(cls) -> list[str]:
"""value list"""
return [i.value for i in cls]

4
backend/video/src/index.py
View File

@ -53,8 +53,8 @@ class SponsorBlock:
print(f"{youtube_id}: get sponsorblock timestamps")
try:
response = requests.get(url, headers=headers, timeout=10)
except requests.ReadTimeout:
print(f"{youtube_id}: sponsorblock API timeout")
except (requests.ReadTimeout, requests.ConnectionError) as err:
print(f"{youtube_id}: sponsorblock API error: {str(err)}")
return False
if not response.ok:

12
backend/video/src/query_building.py
View File

@ -27,20 +27,20 @@ class QueryBuilder:
must_list = []
channel = self.request_params.get("channel")
if channel:
must_list.append({"match": {"channel.channel_id": channel[0]}})
must_list.append({"match": {"channel.channel_id": channel}})
playlist = self.request_params.get("playlist")
if playlist:
must_list.append({"match": {"playlist.keyword": playlist[0]}})
must_list.append({"match": {"playlist.keyword": playlist}})
watch = self.request_params.get("watch")
if watch:
watch_must_list = self.parse_watch(watch[0])
watch_must_list = self.parse_watch(watch)
must_list.append(watch_must_list)
video_type = self.request_params.get("type")
if video_type:
type_list_list = self.parse_type(video_type[0])
type_list_list = self.parse_type(video_type)
must_list.append(type_list_list)
query = {"bool": {"must": must_list}}
@ -88,14 +88,12 @@ class QueryBuilder:
if not sort:
return None
sort = sort[0]
if not hasattr(SortEnum, sort.upper()):
raise ValueError(f"'{sort}' not in SortEnum")
sort_field = getattr(SortEnum, sort.upper()).value
order = self.request_params.get("order", ["desc"])
order = order[0]
order = self.request_params.get("order", "desc")
if not hasattr(OrderEnum, order.upper()):
raise ValueError(f"'{order}' not in OrderEnum")

24
backend/video/tests/test_src/test_query_building.py
View File

@ -15,12 +15,12 @@ def test_build_data():
"""test for correct key building"""
qb = QueryBuilder(
user_id=1,
channel=["test_channel"],
playlist=["test_playlist"],
watch=["watched"],
type=["videos"],
sort=["published"],
order=["desc"],
channel="test_channel",
playlist="test_playlist",
watch="watched",
type="videos",
sort="published",
order="desc",
)
result = qb.build_data()
assert "query" in result
@ -30,7 +30,7 @@ def test_build_data():
def test_parse_watch():
"""watched query building"""
qb = QueryBuilder(user_id=1, watch=["watched"])
qb = QueryBuilder(user_id=1, watch="watched")
result = qb.parse_watch("watched")
assert result == {"match": {"player.watched": True}}
@ -43,7 +43,7 @@ def test_parse_watch():
def test_parse_type():
"""test type is parsed"""
qb = QueryBuilder(user_id=1, type=["videos"])
qb = QueryBuilder(user_id=1, type="videos")
with pytest.raises(ValueError):
qb.parse_type("invalid")
@ -53,16 +53,14 @@ def test_parse_type():
def test_parse_sort():
"""test sort and order"""
qb = QueryBuilder(user_id=1, sort=["views"], order=["desc"])
qb = QueryBuilder(user_id=1, sort="views", order="desc")
result = qb.parse_sort()
assert result == {"sort": [{"stats.view_count": {"order": "desc"}}]}
with pytest.raises(ValueError):
qb = QueryBuilder(user_id=1, sort=["invalid"])
qb = QueryBuilder(user_id=1, sort="invalid")
qb.parse_sort()
with pytest.raises(ValueError):
qb = QueryBuilder(
user_id=1, sort=["stats.view_count"], order=["invalid"]
)
qb = QueryBuilder(user_id=1, sort="stats.view_count", order="invalid")
qb.parse_sort()

193
backend/video/views.py
View File

@ -1,11 +1,22 @@
"""all API views for video endpoints"""
from common.serializers import ErrorResponseSerializer
from common.src.helper import calc_is_watched
from common.src.ta_redis import RedisArchivist
from common.src.watched import WatchState
from common.views_base import AdminWriteOnly, ApiBaseView
from drf_spectacular.utils import OpenApiResponse, extend_schema
from playlist.src.index import YoutubePlaylist
from rest_framework.response import Response
from video.serializers import (
CommentItemSerializer,
PlayerSerializer,
PlaylistNavItemSerializer,
VideoListQuerySerializer,
VideoListSerializer,
VideoProgressUpdateSerializer,
VideoSerializer,
)
from video.src.index import YoutubeVideo
from video.src.query_building import QueryBuilder
@ -24,13 +35,22 @@ class VideoApiListView(ApiBaseView):
search_base = "ta_video/_search/"
@extend_schema(
parameters=[VideoListQuerySerializer()],
responses={
200: VideoListSerializer(),
400: OpenApiResponse(
ErrorResponseSerializer(), description="bad request"
),
},
)
def get(self, request):
"""get request"""
try:
data = QueryBuilder(request.user.id, **request.GET).build_data()
except ValueError as err:
return Response({"error": str(err)}, status=400)
"""get video list"""
query_serializer = VideoListQuerySerializer(data=request.query_params)
query_serializer.is_valid(raise_exception=True)
validated_query = query_serializer.validated_data
data = QueryBuilder(request.user.id, **validated_query).build_data()
if data == {"query": {"bool": {"must": [None]}}}:
# skip empty lookup
return Response([])
@ -38,7 +58,9 @@ class VideoApiListView(ApiBaseView):
self.data = data
self.get_document_list(request, progress_match=request.user.id)
return Response(self.response)
response_serializer = VideoListSerializer(self.response)
return Response(response_serializer.data)
class VideoApiView(ApiBaseView):
@ -49,25 +71,71 @@ class VideoApiView(ApiBaseView):
search_base = "ta_video/_doc/"
permission_classes = [AdminWriteOnly]
@extend_schema(
responses={
200: VideoSerializer(),
404: OpenApiResponse(
ErrorResponseSerializer(), description="video not found"
),
},
)
def get(self, request, video_id):
# pylint: disable=unused-argument
"""get request"""
"""get video"""
self.get_document(video_id, progress_match=request.user.id)
return Response(self.response, status=self.status_code)
if not self.response:
error = ErrorResponseSerializer({"error": "video not found"})
return Response(error.data, status=404)
serializer = VideoSerializer(self.response)
return Response(serializer.data)
@extend_schema(
responses={
204: OpenApiResponse(description="video deleted"),
404: OpenApiResponse(
ErrorResponseSerializer(), description="video not found"
),
}
)
def delete(self, request, video_id):
# pylint: disable=unused-argument
"""delete single video"""
message = {"video": video_id}
"""delete video"""
try:
YoutubeVideo(video_id).delete_media_file()
status_code = 200
message.update({"state": "delete"})
except FileNotFoundError:
status_code = 404
message.update({"state": "not found"})
error = ErrorResponseSerializer({"error": "video not found"})
return Response(error.data, status=404)
return Response(message, status=status_code)
return Response(status=204)
class VideoCommentView(ApiBaseView):
"""resolves to /api/video/<video_id>/comment/
handle video comments
GET: return all comments from video with reply threads
"""
search_base = "ta_comment/_doc/"
@extend_schema(
responses={
200: CommentItemSerializer(),
404: OpenApiResponse(
ErrorResponseSerializer(), description="video not found"
),
}
)
def get(self, request, video_id):
"""get video comments"""
# pylint: disable=unused-argument
self.get_document(video_id)
if self.status_code == 404:
error = ErrorResponseSerializer({"error": "video not found"})
return Response(error.data, status=404)
serializer = CommentItemSerializer(self.response, many=True)
return Response(serializer.data)
class VideoApiNavView(ApiBaseView):
@ -77,26 +145,39 @@ class VideoApiNavView(ApiBaseView):
search_base = "ta_video/_doc/"
@extend_schema(
responses={
200: PlaylistNavItemSerializer(),
404: OpenApiResponse(
ErrorResponseSerializer(), description="video not found"
),
}
)
def get(self, request, video_id):
# pylint: disable=unused-argument
"""get request"""
"""get video playlist nav"""
self.get_document(video_id)
if self.status_code != 200:
return Response(status=self.status_code)
if self.status_code == 404:
error = ErrorResponseSerializer({"error": "video not found"})
return Response(error.data, status=404)
playlist_nav = []
if not self.response["data"].get("playlist"):
if not self.response.get("playlist"):
return Response(playlist_nav)
for playlist_id in self.response["data"]["playlist"]:
for playlist_id in self.response["playlist"]:
playlist = YoutubePlaylist(playlist_id)
playlist.get_from_es()
playlist.build_nav(video_id)
if playlist.nav:
playlist_nav.append(playlist.nav)
return Response(playlist_nav, status=self.status_code)
response_serializer = PlaylistNavItemSerializer(
playlist_nav, many=True
)
return Response(response_serializer.data)
class VideoProgressView(ApiBaseView):
@ -111,19 +192,32 @@ class VideoProgressView(ApiBaseView):
"""redis key"""
return f"{user_id}:progress:{video_id}"
@extend_schema(
request=VideoProgressUpdateSerializer(),
responses={
200: PlayerSerializer(),
404: OpenApiResponse(
ErrorResponseSerializer(), description="video not found"
),
},
)
def post(self, request, video_id):
"""set progress position in redis"""
position = request.data.get("position", 0)
"""set video progress position in redis"""
data_serializer = VideoProgressUpdateSerializer(data=request.data)
data_serializer.is_valid(raise_exception=True)
validated_data = data_serializer.validated_data
self.get_document(video_id)
if self.status_code == 404:
error = ErrorResponseSerializer({"error": "video not found"})
return Response(error.data, status=404)
position = validated_data["position"]
key = self._get_key(request.user.id, video_id)
redis_con = RedisArchivist()
current_progress = redis_con.get_message_dict(key)
if not current_progress:
self.get_document(video_id)
if self.status_code != 200:
return Response(status=self.status_code)
current_progress = self.response["data"]["player"]
current_progress = (
redis_con.get_message_dict(key) or self.response["player"]
)
current_progress.update({"position": position, "youtube_id": video_id})
watched = self._check_watched(request, video_id, current_progress)
@ -134,8 +228,11 @@ class VideoProgressView(ApiBaseView):
current_progress.update({"watched": watched})
redis_con.set_message(key, current_progress, expire=expire)
print(current_progress)
return Response(current_progress)
response_serializer = PlayerSerializer(current_progress)
return Response(response_serializer.data)
def _check_watched(self, request, video_id, current_progress) -> bool:
"""check watched state"""
@ -150,29 +247,17 @@ class VideoProgressView(ApiBaseView):
return watched
@extend_schema(
responses={
204: OpenApiResponse(description="video progress deleted"),
}
)
def delete(self, request, video_id):
"""delete progress position"""
key = self._get_key(request.user.id, video_id)
RedisArchivist().del_message(key)
self.response = {"progress-reset": video_id}
return Response(self.response)
class VideoCommentView(ApiBaseView):
"""resolves to /api/video/<video_id>/comment/
handle video comments
GET: return all comments from video with reply threads
"""
search_base = "ta_comment/_doc/"
def get(self, request, video_id):
"""get video comments"""
# pylint: disable=unused-argument
self.get_document(video_id)
return Response(self.response, status=200)
return Response(status=204)
class VideoSimilarView(ApiBaseView):
@ -182,6 +267,9 @@ class VideoSimilarView(ApiBaseView):
search_base = "ta_video/_search/"
@extend_schema(
responses=VideoSerializer(many=True),
)
def get(self, request, video_id):
"""get similar videos"""
self.data = {
@ -196,4 +284,5 @@ class VideoSimilarView(ApiBaseView):
},
}
self.get_document_list(request, pagination=False)
return Response(self.response, status=200)
serializer = VideoSerializer(self.response["data"], many=True)
return Response(serializer.data)

5
docker_assets/run.sh
View File

@ -8,10 +8,7 @@ python manage.py ta_stop_on_error
# django setup
python manage.py migrate
if [[ -z "$DJANGO_DEBUG" ]]; then
python manage.py collectstatic --noinput -c
fi
python manage.py collectstatic --noinput -c
# ta setup
python manage.py ta_envcheck

1045
frontend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

23
frontend/package.json
View File

@ -11,26 +11,27 @@
"preview": "vite preview"
},
"dependencies": {
"dompurify": "^3.2.3",
"dompurify": "^3.2.4",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.3",
"react-router-dom": "^7.1.5",
"zustand": "^5.0.3"
},
"devDependencies": {
"@types/react": "^19.0.8",
"@types/react-dom": "^19.0.3",
"@typescript-eslint/eslint-plugin": "^8.21.0",
"@typescript-eslint/parser": "^8.21.0",
"@vitejs/plugin-react-swc": "^3.7.2",
"eslint": "^9.19.0",
"@typescript-eslint/eslint-plugin": "^8.24.0",
"@typescript-eslint/parser": "^8.24.0",
"@vitejs/plugin-react-swc": "^3.8.0",
"eslint": "^9.20.1",
"eslint-config-prettier": "^10.0.1",
"eslint-plugin-react-hooks": "^5.1.0",
"eslint-plugin-react-refresh": "^0.4.18",
"globals": "^15.14.0",
"prettier": "3.4.2",
"eslint-plugin-react-refresh": "^0.4.19",
"globals": "^15.15.0",
"prettier": "3.5.1",
"typescript": "^5.7.3",
"typescript-eslint": "^8.21.0",
"vite": "^6.0.11"
"typescript-eslint": "^8.24.0",
"vite": "^6.1.0",
"vite-plugin-checker": "^0.8.0"
}
}

4
frontend/src/api/actions/createCustomPlaylist.ts
View File

@ -1,9 +1,9 @@
import APIClient from '../../functions/APIClient';
const createCustomPlaylist = async (playlistId: string) => {
return APIClient('/api/playlist/', {
return APIClient('/api/playlist/custom/', {
method: 'POST',
body: { data: { create: playlistId } },
body: { playlist_name: playlistId },
});
};

8
frontend/src/api/actions/updateAppsettingsConfig.ts
View File

@ -1,12 +1,10 @@
import APIClient from '../../functions/APIClient';
import { AppSettingsConfigType } from '../loader/loadAppsettingsConfig';
const updateAppsettingsConfig = async (
configKey: string,
configValue: string | boolean | number | null,
) => {
const updateAppsettingsConfig = async (updatedConfig: Partial<AppSettingsConfigType>) => {
return APIClient('/api/appsettings/config/', {
method: 'POST',
body: { [configKey]: configValue },
body: updatedConfig,
});
};

2
frontend/src/api/actions/updateCustomPlaylist.ts
View File

@ -7,7 +7,7 @@ const updateCustomPlaylist = async (
playlistId: string,
videoId: string,
) => {
return APIClient(`/api/playlist/${playlistId}/`, {
return APIClient(`/api/playlist/custom/${playlistId}/`, {
method: 'POST',
body: { action, video_id: videoId },
});

13
frontend/src/api/actions/updateUserConfig.ts
View File

@ -1,17 +1,6 @@
import { SortByType, SortOrderType, ViewLayoutType } from '../../pages/Home';
import APIClient from '../../functions/APIClient';
export type UserMeType = {
id: number;
name: string;
is_superuser: boolean;
is_staff: boolean;
groups: [];
user_permissions: [];
last_login: string;
config: UserConfigType;
};
export type ColourVariants = 'dark.css' | 'light.css' | 'matrix.css' | 'midnight.css';
export type UserConfigType = {
@ -33,7 +22,7 @@ export type UserConfigType = {
const updateUserConfig = async (config: Partial<UserConfigType>): Promise<UserConfigType> => {
return APIClient('/api/user/me/', {
method: 'POST',
body: { config: config },
body: config,
});
};

8
frontend/src/api/loader/loadDownloadAggs.ts
View File

@ -7,11 +7,9 @@ type DownloadAggsBucket = {
};
export type DownloadAggsType = {
channel_downloads: {
doc_count_error_upper_bound: number;
sum_other_doc_count: number;
buckets: DownloadAggsBucket[];
};
doc_count_error_upper_bound: number;
sum_other_doc_count: number;
buckets: DownloadAggsBucket[];
};
const loadDownloadAggs = async (showIgnored: boolean): Promise<DownloadAggsType> => {

28
frontend/src/api/loader/loadPlaylistById.ts
View File

@ -1,6 +1,32 @@
import APIClient from '../../functions/APIClient';
const loadPlaylistById = async (playlistId: string | undefined) => {
export type PlaylistEntryType = {
youtube_id: string;
title: string;
uploader: string;
idx: number;
downloaded: boolean;
};
export type PlaylistType = {
playlist_active: boolean;
playlist_channel: string;
playlist_channel_id: string;
playlist_description: string;
playlist_entries: PlaylistEntryType[];
playlist_id: string;
playlist_last_refresh: string;
playlist_name: string;
playlist_subscribed: boolean;
playlist_thumbnail: string;
playlist_type: string;
_index: string;
_score: number;
};
export type PlaylistResponseType = PlaylistType;
const loadPlaylistById = async (playlistId: string | undefined): Promise<PlaylistResponseType> => {
return APIClient(`/api/playlist/${playlistId}/`);
};

17
frontend/src/api/loader/loadUserAccount.ts Normal file
View File

@ -0,0 +1,17 @@
import APIClient from '../../functions/APIClient';
export type UserAccountType = {
id: number;
name: string;
is_superuser: boolean;
is_staff: boolean;
groups: [];
user_permissions: [];
last_login: string;
};
const loadUserAccount = async (): Promise<UserAccountType> => {
return APIClient('/api/user/account/');
};
export default loadUserAccount;

4
frontend/src/api/loader/loadUserConfig.ts
View File

@ -1,7 +1,7 @@
import { UserMeType } from '../actions/updateUserConfig';
import { UserConfigType } from '../actions/updateUserConfig';
import APIClient from '../../functions/APIClient';
const loadUserMeConfig = async (): Promise<UserMeType> => {
const loadUserMeConfig = async (): Promise<UserConfigType> => {
return APIClient('/api/user/me/');
};

2
frontend/src/components/ChannelList.tsx
View File

@ -16,7 +16,7 @@ type ChannelListProps = {
const ChannelList = ({ channelList, refreshChannelList }: ChannelListProps) => {
const { userConfig } = useUserConfigStore();
const viewLayout = userConfig.config.view_style_channel;
const viewLayout = userConfig.view_style_channel;
if (!channelList || channelList.length === 0) {
return <p>No channels found.</p>;

16
frontend/src/components/DownloadListItem.tsx
View File

@ -1,3 +1,4 @@
import { useState } from 'react';
import { Link } from 'react-router-dom';
import Download from '../pages/Download';
import Routes from '../configuration/routes/RouteList';
@ -5,9 +6,9 @@ import formatDate from '../functions/formatDates';
import Button from './Button';
import deleteDownloadById from '../api/actions/deleteDownloadById';
import updateDownloadQueueStatusById from '../api/actions/updateDownloadQueueStatusById';
import { useState } from 'react';
import getApiUrl from '../configuration/getApiUrl';
import { useUserConfigStore } from '../stores/UserConfigStore';
import defaultVideoThumb from '/img/default-video-thumb.jpg';
type DownloadListItemProps = {
download: Download;
@ -16,8 +17,8 @@ type DownloadListItemProps = {
const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {
const { userConfig } = useUserConfigStore();
const view = userConfig.config.view_style_downloads;
const showIgnored = userConfig.config.show_ignored_only;
const view = userConfig.view_style_downloads;
const showIgnored = userConfig.show_ignored_only;
const [hideDownload, setHideDownload] = useState(false);
@ -25,7 +26,14 @@ const DownloadListItem = ({ download, setRefresh }: DownloadListItemProps) => {
<div className={`video-item ${view}`} id={`dl-${download.youtube_id}`}>
<div className={`video-thumb-wrap ${view}`}>
<div className="video-thumb">
<img src={`${getApiUrl()}${download.vid_thumb_url}`} alt="video_thumb" />
<img
src={`${getApiUrl()}${download.vid_thumb_url}`}
alt="video_thumb"
onError={({ currentTarget }) => {
currentTarget.onerror = null; // prevents looping
currentTarget.src = defaultVideoThumb;
}}
/>
<div className="video-tags">
{showIgnored && <span>ignored</span>}

17
frontend/src/components/EmbeddableVideoPlayer.tsx
View File

@ -12,6 +12,7 @@ import formatNumbers from '../functions/formatNumbers';
import { Link, useSearchParams } from 'react-router-dom';
import Routes from '../configuration/routes/RouteList';
import loadPlaylistById from '../api/loader/loadPlaylistById';
import { useAppSettingsStore } from '../stores/AppSettingsStore';
type Playlist = {
id: string;
@ -25,6 +26,7 @@ type EmbeddableVideoPlayerProps = {
const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
const inlinePlayerRef = useRef<HTMLDivElement>(null);
const { appSettingsConfig } = useAppSettingsStore();
const [, setSearchParams] = useSearchParams();
@ -35,16 +37,16 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
useEffect(() => {
(async () => {
if (refresh || videoId !== videoResponse?.data.youtube_id) {
if (refresh || videoId !== videoResponse?.youtube_id) {
const videoResponse = await loadVideoById(videoId);
const playlistIds = videoResponse.data.playlist;
const playlistIds = videoResponse.playlist;
if (playlistIds !== undefined) {
const playlists = await Promise.all(
playlistIds.map(async playlistid => {
const playlistResponse = await loadPlaylistById(playlistid);
return playlistResponse.data;
return playlistResponse;
}),
);
@ -69,6 +71,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
setRefresh(false);
}
})();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [videoId, refresh]);
useEffect(() => {
@ -79,7 +82,7 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
return <div ref={inlinePlayerRef} className="player-wrapper" />;
}
const video = videoResponse.data;
const video = videoResponse;
const name = video.title;
const channelId = video.channel.channel_id;
const channelName = video.channel.channel_name;
@ -88,10 +91,10 @@ const EmbeddableVideoPlayer = ({ videoId }: EmbeddableVideoPlayerProps) => {
const views = formatNumbers(video.stats.view_count);
const hasLikes = video.stats.like_count;
const likes = formatNumbers(video.stats.like_count);
const hasDislikes = video.stats.dislike_count > 0 && videoResponse.config.downloads.integrate_ryd;
const hasDislikes = video.stats.dislike_count > 0 && appSettingsConfig.downloads.integrate_ryd;
const dislikes = formatNumbers(video.stats.dislike_count);
const config = videoResponse.config;
const cast = config.enable_cast;
// const config = videoResponse.config;
const cast = false; // config.enable_cast;
return (
<>

38
frontend/src/components/Filterbar.tsx
View File

@ -7,6 +7,7 @@ import iconListView from '/img/icon-listview.svg';
import { SortByType, SortOrderType } from '../pages/Home';
import { useUserConfigStore } from '../stores/UserConfigStore';
import { ViewStyles } from '../configuration/constants/ViewStyle';
import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
type FilterbarProps = {
hideToggleText: string;
@ -15,9 +16,14 @@ type FilterbarProps = {
};
const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: FilterbarProps) => {
const { userConfig, setPartialConfig } = useUserConfigStore();
const { userConfig, setUserConfig } = useUserConfigStore();
const [showHidden, setShowHidden] = useState(false);
const isGridView = userConfig.config.view_style_home === ViewStyles.grid;
const isGridView = userConfig.view_style_home === ViewStyles.grid;
const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
const updatedUserConfig = await updateUserConfig(config);
setUserConfig(updatedUserConfig);
};
return (
<div className="view-controls three">
@ -27,13 +33,13 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
<input
id="hide_watched"
type="checkbox"
checked={userConfig.config.hide_watched}
checked={userConfig.hide_watched}
onChange={() => {
setPartialConfig({ hide_watched: !userConfig.config.hide_watched });
handleUserConfigUpdate({ hide_watched: !userConfig.hide_watched });
}}
/>
{userConfig.config.hide_watched ? (
{userConfig.hide_watched ? (
<label htmlFor="" className="onbtn">
On
</label>
@ -52,9 +58,9 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
<select
name="sort_by"
id="sort"
value={userConfig.config.sort_by}
value={userConfig.sort_by}
onChange={event => {
setPartialConfig({ sort_by: event.target.value as SortByType });
handleUserConfigUpdate({ sort_by: event.target.value as SortByType });
}}
>
<option value="published">date published</option>
@ -67,9 +73,9 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
<select
name="sort_order"
id="sort-order"
value={userConfig.config.sort_order}
value={userConfig.sort_order}
onChange={event => {
setPartialConfig({ sort_order: event.target.value as SortOrderType });
handleUserConfigUpdate({ sort_order: event.target.value as SortOrderType });
}}
>
<option value="asc">asc</option>
@ -90,22 +96,22 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
/>
)}
{userConfig.config.grid_items !== undefined && isGridView && (
{userConfig.grid_items !== undefined && isGridView && (
<div className="grid-count">
{userConfig.config.grid_items < 7 && (
{userConfig.grid_items < 7 && (
<img
src={iconAdd}
onClick={() => {
setPartialConfig({ grid_items: userConfig.config.grid_items + 1 });
handleUserConfigUpdate({ grid_items: userConfig.grid_items + 1 });
}}
alt="grid plus row"
/>
)}
{userConfig.config.grid_items > 3 && (
{userConfig.grid_items > 3 && (
<img
src={iconSubstract}
onClick={() => {
setPartialConfig({ grid_items: userConfig.config.grid_items - 1 });
handleUserConfigUpdate({ grid_items: userConfig.grid_items - 1 });
}}
alt="grid minus row"
/>
@ -115,14 +121,14 @@ const Filterbar = ({ hideToggleText, viewStyleName, showSort = true }: Filterbar
<img
src={iconGridView}
onClick={() => {
setPartialConfig({ [viewStyleName]: 'grid' });
handleUserConfigUpdate({ [viewStyleName]: 'grid' });
}}
alt="grid view"
/>
<img
src={iconListView}
onClick={() => {
setPartialConfig({ [viewStyleName]: 'list' });
handleUserConfigUpdate({ [viewStyleName]: 'list' });
}}
alt="list view"
/>

12
frontend/src/components/GoogleCast.tsx
View File

@ -93,8 +93,10 @@ const GoogleCast = ({ video, setRefresh, onWatchStateChanged }: GoogleCastProps)
const [isConnected, setIsConnected] = useState(false);
const setup = useCallback(() => {
const cast = globalThis.cast;
const chrome = globalThis.chrome;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cast = (globalThis as any).cast;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const chrome = (globalThis as any).chrome;
cast.framework.CastContext.getInstance().setOptions({
receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID, // Use built in receiver app on cast device, see https://developers.google.com/cast/docs/styled_receiver if you want to be able to add a theme, splash screen or watermark. Has a $5 one time fee.
@ -132,8 +134,10 @@ const GoogleCast = ({ video, setRefresh, onWatchStateChanged }: GoogleCastProps)
}, [setRefresh, video]);
const startPlayback = useCallback(() => {
const chrome = globalThis.chrome;
const cast = globalThis.cast;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const chrome = (globalThis as any).chrome;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cast = (globalThis as any).cast;
const castSession = cast.framework.CastContext.getInstance().getCurrentSession();
const mediaUrl = video?.media_url;

4
frontend/src/components/PlaylistList.tsx
View File

@ -1,11 +1,11 @@
import { Link } from 'react-router-dom';
import Routes from '../configuration/routes/RouteList';
import { PlaylistType } from '../pages/Playlist';
import updatePlaylistSubscription from '../api/actions/updatePlaylistSubscription';
import formatDate from '../functions/formatDates';
import Button from './Button';
import PlaylistThumbnail from './PlaylistThumbnail';
import { useUserConfigStore } from '../stores/UserConfigStore';
import { PlaylistType } from '../api/loader/loadPlaylistById';
type PlaylistListProps = {
playlistList: PlaylistType[] | undefined;
@ -14,7 +14,7 @@ type PlaylistListProps = {
const PlaylistList = ({ playlistList, setRefresh }: PlaylistListProps) => {
const { userConfig } = useUserConfigStore();
const viewLayout = userConfig.config.view_style_playlist;
const viewLayout = userConfig.view_style_playlist;
if (!playlistList || playlistList.length === 0) {
return <p>No playlists found.</p>;

12
frontend/src/components/VideoListItem.tsx
View File

@ -45,10 +45,14 @@ const VideoListItem = ({
>
<div className={`video-thumb-wrap ${viewLayout}`}>
<div className="video-thumb">
<picture>
<img src={`${getApiUrl()}${video.vid_thumb_url}`} alt="video-thumb" />
<source srcSet={defaultVideoThumb} />
</picture>
<img
src={`${getApiUrl()}${video.vid_thumb_url}`}
alt="video-thumb"
onError={({ currentTarget }) => {
currentTarget.onerror = null; // prevents looping
currentTarget.src = defaultVideoThumb;
}}
/>
{video.player.progress && (
<div

17
frontend/src/components/VideoPlayer.tsx
View File

@ -59,7 +59,6 @@ const Subtitles = ({ subtitles }: SubtitlesProp) => {
const handleTimeUpdate =
(
youtubeId: string,
duration: number,
watched: boolean,
sponsorBlock?: SponsorBlockType,
setSponsorSegmentSkipped?: Dispatch<SetStateAction<SponsorSegmentsSkippedType>>,
@ -142,15 +141,14 @@ const VideoPlayer = ({
const arrowRightPressed = useKeyPress('ArrowRight');
const arrowLeftPressed = useKeyPress('ArrowLeft');
const videoId = video.data.youtube_id;
const videoUrl = video.data.media_url;
const videoThumbUrl = video.data.vid_thumb_url;
const watched = video.data.player.watched;
const duration = video.data.player.duration;
const videoSubtitles = video.data.subtitles;
const videoId = video.youtube_id;
const videoUrl = video.media_url;
const videoThumbUrl = video.vid_thumb_url;
const watched = video.player.watched;
const duration = video.player.duration;
const videoSubtitles = video.subtitles;
let videoSrcProgress =
Number(video.data.player?.position) > 0 ? Number(video.data.player?.position) : '';
let videoSrcProgress = Number(video.player?.position) > 0 ? Number(video.player?.position) : '';
if (searchParamVideoProgress !== null) {
videoSrcProgress = searchParamVideoProgress;
@ -337,7 +335,6 @@ const VideoPlayer = ({
}}
onTimeUpdate={handleTimeUpdate(
videoId,
duration,
watched,
sponsorBlock,
setSkippedSegments,

2
frontend/src/configuration/colours/useColours.ts
View File

@ -9,7 +9,7 @@ export const ColourConstant = {
const useColours = () => {
const { userConfig } = useUserConfigStore();
const stylesheet = userConfig?.config.stylesheet;
const stylesheet = userConfig?.stylesheet;
switch (stylesheet) {
case ColourConstant.Dark:

9
frontend/src/functions/APIClient.ts
View File

@ -55,8 +55,15 @@ const APIClient = async (
throw new Error('Forbidden: Access denied.');
}
// Try parsing response data
let data;
// expected empty response
if (response.status === 204) {
data = null;
return data;
}
// Try parsing response data
try {
data = await response.json();
} catch (error) {

6
frontend/src/functions/useIsAdmin.ts
View File

@ -1,8 +1,8 @@
import { useUserConfigStore } from '../stores/UserConfigStore';
import { useUserAccountStore } from '../stores/UserAccountStore';
const useIsAdmin = () => {
const { userConfig } = useUserConfigStore();
const isAdmin = userConfig?.is_staff || userConfig?.is_superuser;
const { userAccount } = useUserAccountStore();
const isAdmin = userAccount?.is_staff || userAccount?.is_superuser;
return isAdmin;
};

6
frontend/src/main.tsx
View File

@ -25,6 +25,8 @@ import ChannelVideo from './pages/ChannelVideo';
import ChannelPlaylist from './pages/ChannelPlaylist';
import ChannelAbout from './pages/ChannelAbout';
import Download from './pages/Download';
import loadUserAccount from './api/loader/loadUserAccount';
import loadAppsettingsConfig from './api/loader/loadAppsettingsConfig';
const router = createBrowserRouter(
[
@ -41,8 +43,10 @@ const router = createBrowserRouter(
const authData = await auth.json();
const userConfig = await loadUserMeConfig();
const userAccount = await loadUserAccount();
const appSettings = await loadAppsettingsConfig();
return { userConfig, auth: authData };
return { userConfig, userAccount, appSettings, auth: authData };
},
element: <Base />,
errorElement: <ErrorPage />,

20
frontend/src/pages/Base.tsx
View File

@ -1,13 +1,17 @@
import { Outlet, useLoaderData, useLocation, useSearchParams } from 'react-router-dom';
import Footer from '../components/Footer';
import useColours from '../configuration/colours/useColours';
import { UserMeType } from '../api/actions/updateUserConfig';
import { UserConfigType } from '../api/actions/updateUserConfig';
import { useEffect, useState } from 'react';
import Navigation from '../components/Navigation';
import { useAuthStore } from '../stores/AuthDataStore';
import { useUserConfigStore } from '../stores/UserConfigStore';
import { useUserAccountStore } from '../stores/UserAccountStore';
import { UserAccountType } from '../api/loader/loadUserAccount';
import { useAppSettingsStore } from '../stores/AppSettingsStore';
import { AppSettingsConfigType } from '../api/loader/loadAppsettingsConfig';
export type TaUpdateType = {
type TaUpdateType = {
version?: string;
is_breaking?: boolean;
};
@ -20,7 +24,9 @@ export type AuthenticationType = {
};
type BaseLoaderData = {
userConfig: UserMeType;
userConfig: UserConfigType;
userAccount: UserAccountType;
appSettings: AppSettingsConfigType;
auth: AuthenticationType;
};
@ -32,7 +38,10 @@ export type OutletContextType = {
const Base = () => {
const { setAuth } = useAuthStore();
const { setUserConfig } = useUserConfigStore();
const { userConfig, auth } = useLoaderData() as BaseLoaderData;
const { setUserAccount } = useUserAccountStore();
const { setAppSettingsConfig } = useAppSettingsStore();
const { userConfig, userAccount, appSettings, auth } = useLoaderData() as BaseLoaderData;
const location = useLocation();
@ -46,6 +55,9 @@ const Base = () => {
useEffect(() => {
setAuth(auth);
setUserConfig(userConfig);
setUserAccount(userAccount);
setAppSettingsConfig(appSettings);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);

8
frontend/src/pages/ChannelAbout.tsx
View File

@ -55,7 +55,7 @@ const ChannelAbout = () => {
const [pageSizeShorts, setPageSizeShorts] = useState<number | null>(null);
const [pageSizeStreams, setPageSizeStreams] = useState<number | null>(null);
const channel = channelResponse?.data;
const channel = channelResponse;
useEffect(() => {
(async () => {
@ -226,7 +226,7 @@ const ChannelAbout = () => {
</div>
)}
{channel.channel_tags && (
{channel.channel_tags.length > 0 && (
<div className="description-box">
<div className="video-tag-box">
{channel.channel_tags.map(tag => {
@ -244,7 +244,7 @@ const ChannelAbout = () => {
<div className="info-box">
<div className="info-box-item">
<h2>Channel Customization</h2>
{userConfig.config.show_help_text && (
{userConfig.show_help_text && (
<div className="help-text">
<ul>
<li>Overwrite the download format over the format set globally.</li>
@ -334,7 +334,7 @@ const ChannelAbout = () => {
</div>
<div className="info-box-item">
<h2>Page Size Overrides</h2>
{userConfig.config.show_help_text && (
{userConfig.show_help_text && (
<div className="help-text">
<p>
Disable standard videos, shorts, or streams for this channel by setting their

8
frontend/src/pages/ChannelBase.tsx
View File

@ -1,7 +1,6 @@
import { Link, Outlet, useOutletContext, useParams } from 'react-router-dom';
import Routes from '../configuration/routes/RouteList';
import { ChannelType } from './Channels';
import { ConfigType } from './Home';
import { OutletContextType } from './Base';
import Notifications from '../components/Notifications';
import { useEffect, useState } from 'react';
@ -14,10 +13,7 @@ type ChannelParams = {
channelId: string;
};
export type ChannelResponseType = {
data: ChannelType;
config: ConfigType;
};
export type ChannelResponseType = ChannelType;
const ChannelBase = () => {
const { channelId } = useParams() as ChannelParams;
@ -28,7 +24,7 @@ const ChannelBase = () => {
const [channelNav, setChannelNav] = useState<ChannelNavResponseType>();
const [startNotification, setStartNotification] = useState(false);
const channel = channelResponse?.data;
const channel = channelResponse;
const { has_streams, has_shorts, has_playlists, has_pending } = channelNav || {};
useEffect(() => {

18
frontend/src/pages/ChannelPlaylist.tsx
View File

@ -10,10 +10,11 @@ import { PlaylistsResponseType } from './Playlists';
import iconGridView from '/img/icon-gridview.svg';
import iconListView from '/img/icon-listview.svg';
import { useUserConfigStore } from '../stores/UserConfigStore';
import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
const ChannelPlaylist = () => {
const { channelId } = useParams();
const { userConfig, setPartialConfig } = useUserConfigStore();
const { userConfig, setUserConfig } = useUserConfigStore();
const { currentPage, setCurrentPage } = useOutletContext() as OutletContextType;
const [refreshPlaylists, setRefreshPlaylists] = useState(false);
@ -23,8 +24,13 @@ const ChannelPlaylist = () => {
const playlistList = playlistsResponse?.data;
const pagination = playlistsResponse?.paginate;
const view = userConfig.config.view_style_playlist;
const showSubedOnly = userConfig.config.show_subed_only;
const view = userConfig.view_style_playlist;
const showSubedOnly = userConfig.show_subed_only;
const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
const updatedUserConfig = await updateUserConfig(config);
setUserConfig(updatedUserConfig);
};
useEffect(() => {
(async () => {
@ -52,7 +58,7 @@ const ChannelPlaylist = () => {
<input
checked={showSubedOnly}
onChange={() => {
setPartialConfig({ show_subed_only: !showSubedOnly });
handleUserConfigUpdate({ show_subed_only: !showSubedOnly });
setRefreshPlaylists(true);
}}
type="checkbox"
@ -73,14 +79,14 @@ const ChannelPlaylist = () => {
<img
src={iconGridView}
onClick={() => {
setPartialConfig({ view_style_playlist: 'grid' });
handleUserConfigUpdate({ view_style_playlist: 'grid' });
}}
alt="grid view"
/>
<img
src={iconListView}
onClick={() => {
setPartialConfig({ view_style_playlist: 'list' });
handleUserConfigUpdate({ view_style_playlist: 'list' });
}}
alt="list view"
/>

20
frontend/src/pages/ChannelVideo.tsx
View File

@ -42,17 +42,17 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
const [videoResponse, setVideoReponse] = useState<VideoListByFilterResponseType>();
const [videoAggsResponse, setVideoAggsResponse] = useState<ChannelAggsType>();
const channel = channelResponse?.data;
const channel = channelResponse;
const videoList = videoResponse?.data;
const pagination = videoResponse?.paginate;
const hasVideos = videoResponse?.data?.length !== 0;
const showEmbeddedVideo = videoId !== null;
const view = userConfig.config.view_style_home;
const view = userConfig.view_style_home;
const isGridView = view === ViewStyles.grid;
const gridView = isGridView ? `boxed-${userConfig.config.grid_items}` : '';
const gridViewGrid = isGridView ? `grid-${userConfig.config.grid_items}` : '';
const gridView = isGridView ? `boxed-${userConfig.grid_items}` : '';
const gridViewGrid = isGridView ? `grid-${userConfig.grid_items}` : '';
useEffect(() => {
(async () => {
@ -60,9 +60,9 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
const videos = await loadVideoListByFilter({
channel: channelId,
page: currentPage,
watch: userConfig.config.hide_watched ? 'unwatched' : undefined,
sort: userConfig.config.sort_by,
order: userConfig.config.sort_order,
watch: userConfig.hide_watched ? 'unwatched' : undefined,
sort: userConfig.sort_by,
order: userConfig.sort_order,
type: videoType,
});
const channelAggs = await loadChannelAggs(channelId);
@ -74,9 +74,9 @@ const ChannelVideo = ({ videoType }: ChannelVideoProps) => {
})();
}, [
refresh,
userConfig.config.sort_by,
userConfig.config.sort_order,
userConfig.config.hide_watched,
userConfig.sort_by,
userConfig.sort_order,
userConfig.hide_watched,
currentPage,
channelId,
pagination?.current_page,

29
frontend/src/pages/Channels.tsx
View File

@ -14,6 +14,7 @@ import Button from '../components/Button';
import updateBulkChannelSubscriptions from '../api/actions/updateBulkChannelSubscriptions';
import useIsAdmin from '../functions/useIsAdmin';
import { useUserConfigStore } from '../stores/UserConfigStore';
import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
type ChannelOverwritesType = {
download_format?: string;
@ -48,7 +49,7 @@ type ChannelsListResponse = {
};
const Channels = () => {
const { userConfig, setPartialConfig } = useUserConfigStore();
const { userConfig, setUserConfig } = useUserConfigStore();
const { currentPage, setCurrentPage } = useOutletContext() as OutletContextType;
const isAdmin = useIsAdmin();
@ -63,18 +64,20 @@ const Channels = () => {
// const channelCount = pagination?.total_hits;
const hasChannels = channels?.length !== 0;
const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
const updatedUserConfig = await updateUserConfig(config);
setUserConfig(updatedUserConfig);
};
useEffect(() => {
(async () => {
const channelListResponse = await loadChannelList(
currentPage,
userConfig.config.show_subed_only,
);
const channelListResponse = await loadChannelList(currentPage, userConfig.show_subed_only);
setChannelListResponse(channelListResponse);
setShowNotification(false);
setRefresh(false);
})();
}, [refresh, userConfig.config.show_subed_only, currentPage, pagination?.current_page]);
}, [refresh, userConfig.show_subed_only, currentPage, pagination?.current_page]);
return (
<>
@ -145,18 +148,18 @@ const Channels = () => {
<input
id="show_subed_only"
onChange={async () => {
setPartialConfig({ show_subed_only: !userConfig.config.show_subed_only });
handleUserConfigUpdate({ show_subed_only: !userConfig.show_subed_only });
setRefresh(true);
}}
type="checkbox"
checked={userConfig.config.show_subed_only}
checked={userConfig.show_subed_only}
/>
{!userConfig.config.show_subed_only && (
{!userConfig.show_subed_only && (
<label htmlFor="" className="ofbtn">
Off
</label>
)}
{userConfig.config.show_subed_only && (
{userConfig.show_subed_only && (
<label htmlFor="" className="onbtn">
On
</label>
@ -167,7 +170,7 @@ const Channels = () => {
<img
src={iconGridView}
onClick={() => {
setPartialConfig({ view_style_channel: 'grid' });
handleUserConfigUpdate({ view_style_channel: 'grid' });
}}
data-origin="channel"
data-value="grid"
@ -176,7 +179,7 @@ const Channels = () => {
<img
src={iconListView}
onClick={() => {
setPartialConfig({ view_style_channel: 'list' });
handleUserConfigUpdate({ view_style_channel: 'list' });
}}
data-origin="channel"
data-value="list"
@ -186,7 +189,7 @@ const Channels = () => {
</div>
{/* {hasChannels && <h2>Total channels: {channelCount}</h2>} */}
<div className={`channel-list ${userConfig.config.view_style_channel}`}>
<div className={`channel-list ${userConfig.view_style_channel}`}>
{!hasChannels && <h2>No channels found...</h2>}
{hasChannels && <ChannelList channelList={channels} refreshChannelList={setRefresh} />}

27
frontend/src/pages/Download.tsx
View File

@ -19,6 +19,7 @@ import Button from '../components/Button';
import DownloadListItem from '../components/DownloadListItem';
import loadDownloadAggs, { DownloadAggsType } from '../api/loader/loadDownloadAggs';
import { useUserConfigStore } from '../stores/UserConfigStore';
import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
type Download = {
auto_start: boolean;
@ -46,7 +47,7 @@ export type DownloadResponseType = {
const Download = () => {
const [searchParams, setSearchParams] = useSearchParams();
const { userConfig, setPartialConfig } = useUserConfigStore();
const { userConfig, setUserConfig } = useUserConfigStore();
const { currentPage, setCurrentPage } = useOutletContext() as OutletContextType;
const channelFilterFromUrl = searchParams.get('channel');
@ -65,8 +66,7 @@ const Download = () => {
const downloadList = downloadResponse?.data;
const pagination = downloadResponse?.paginate;
const channelDownloads = downloadAggsResponse?.channel_downloads;
const channelAggsList = channelDownloads?.buckets;
const channelAggsList = downloadAggsResponse?.buckets;
const downloadCount = pagination?.total_hits;
@ -75,13 +75,18 @@ const Download = () => {
? downloadResponse?.data[0].channel_name
: '';
const view = userConfig.config.view_style_downloads;
const gridItems = userConfig.config.grid_items;
const showIgnored = userConfig.config.show_ignored_only;
const view = userConfig.view_style_downloads;
const gridItems = userConfig.grid_items;
const showIgnored = userConfig.show_ignored_only;
const isGridView = view === ViewStyles.grid;
const gridView = isGridView ? `boxed-${gridItems}` : '';
const gridViewGrid = isGridView ? `grid-${gridItems}` : '';
const handleUserConfigUpdate = async (config: Partial<UserConfigType>) => {
const updatedUserConfig = await updateUserConfig(config);
setUserConfig(updatedUserConfig);
};
useEffect(() => {
(async () => {
const videos = await loadDownloadQueue(currentPage, channelFilterFromUrl, showIgnored);
@ -208,7 +213,7 @@ const Download = () => {
<input
id="showIgnored"
onChange={() => {
setPartialConfig({ show_ignored_only: !showIgnored });
handleUserConfigUpdate({ show_ignored_only: !showIgnored });
setRefresh(true);
}}
type="checkbox"
@ -263,7 +268,7 @@ const Download = () => {
<img
src={iconAdd}
onClick={() => {
setPartialConfig({ grid_items: gridItems + 1 });
handleUserConfigUpdate({ grid_items: gridItems + 1 });
}}
alt="grid plus row"
/>
@ -272,7 +277,7 @@ const Download = () => {
<img
src={iconSubstract}
onClick={() => {
setPartialConfig({ grid_items: gridItems - 1 });
handleUserConfigUpdate({ grid_items: gridItems - 1 });
}}
alt="grid minus row"
/>
@ -283,14 +288,14 @@ const Download = () => {
<img
src={iconGridView}
onClick={() => {
setPartialConfig({ view_style_downloads: 'grid' });
handleUserConfigUpdate({ view_style_downloads: 'grid' });
}}
alt="grid view"
/>
<img
src={iconListView}
onClick={() => {
setPartialConfig({ view_style_downloads: 'list' });
handleUserConfigUpdate({ view_style_downloads: 'list' });
}}
alt="list view"
/>

30
frontend/src/pages/Home.tsx
View File

@ -60,7 +60,7 @@ export type VideoType = {
player: PlayerType;
published: string;
sponsorblock?: SponsorBlockType;
playlist?: string[];
playlist: string[];
stats: StatsType;
streams: StreamType[];
subtitles: Subtitles[];
@ -108,8 +108,6 @@ const Home = () => {
const [searchParams] = useSearchParams();
const videoId = searchParams.get('videoId');
const userMeConfig = userConfig.config;
const [refreshVideoList, setRefreshVideoList] = useState(false);
const [videoResponse, setVideoReponse] = useState<VideoListByFilterResponseType>();
@ -123,17 +121,17 @@ const Home = () => {
const hasVideos = videoResponse?.data?.length !== 0;
const showEmbeddedVideo = videoId !== null;
const isGridView = userMeConfig.view_style_home === ViewStyles.grid;
const gridView = isGridView ? `boxed-${userMeConfig.grid_items}` : '';
const gridViewGrid = isGridView ? `grid-${userMeConfig.grid_items}` : '';
const isGridView = userConfig.view_style_home === ViewStyles.grid;
const gridView = isGridView ? `boxed-${userConfig.grid_items}` : '';
const gridViewGrid = isGridView ? `grid-${userConfig.grid_items}` : '';
useEffect(() => {
(async () => {
const videos = await loadVideoListByFilter({
page: currentPage,
watch: userMeConfig.hide_watched ? 'unwatched' : undefined,
sort: userMeConfig.sort_by,
order: userMeConfig.sort_order,
watch: userConfig.hide_watched ? 'unwatched' : undefined,
sort: userConfig.sort_by,
order: userConfig.sort_order,
});
try {
@ -150,9 +148,9 @@ const Home = () => {
})();
}, [
refreshVideoList,
userMeConfig.sort_by,
userMeConfig.sort_order,
userMeConfig.hide_watched,
userConfig.sort_by,
userConfig.sort_order,
userConfig.hide_watched,
currentPage,
pagination?.current_page,
showEmbeddedVideo,
@ -171,10 +169,10 @@ const Home = () => {
<div className="title-bar">
<h1>Continue Watching</h1>
</div>
<div className={`video-list ${userMeConfig.view_style_home} ${gridViewGrid}`}>
<div className={`video-list ${userConfig.view_style_home} ${gridViewGrid}`}>
<VideoList
videoList={continueVideos}
viewLayout={userMeConfig.view_style_home}
viewLayout={userConfig.view_style_home}
refreshVideoList={setRefreshVideoList}
/>
</div>
@ -189,7 +187,7 @@ const Home = () => {
</div>
<div className={`boxed-content ${gridView}`}>
<div className={`video-list ${userMeConfig.view_style_home} ${gridViewGrid}`}>
<div className={`video-list ${userConfig.view_style_home} ${gridViewGrid}`}>
{!hasVideos && (
<>
<h2>No videos found...</h2>
@ -204,7 +202,7 @@ const Home = () => {
{hasVideos && (
<VideoList
videoList={videoList}
viewLayout={userMeConfig.view_style_home}
viewLayout={userConfig.view_style_home}
refreshVideoList={setRefreshVideoList}
/>
)}

2
frontend/src/pages/Login.tsx
View File

@ -22,7 +22,7 @@ const Login = () => {
const loginResponse = await signIn(username, password, saveLogin);
const signedIn = loginResponse.status === 200;
const signedIn = loginResponse.status === 204;
if (signedIn) {
navigate(Routes.Home);

47
frontend/src/pages/Playlist.tsx
View File

@ -1,10 +1,9 @@
import { useEffect, useState } from 'react';
import { Link, useNavigate, useOutletContext, useParams, useSearchParams } from 'react-router-dom';
import loadPlaylistById from '../api/loader/loadPlaylistById';
import loadPlaylistById, { PlaylistResponseType } from '../api/loader/loadPlaylistById';
import { OutletContextType } from './Base';
import { ConfigType, VideoType } from './Home';
import { VideoType } from './Home';
import Filterbar from '../components/Filterbar';
import { PlaylistEntryType } from './Playlists';
import loadChannelById from '../api/loader/loadChannelById';
import VideoList from '../components/VideoList';
import Pagination, { PaginationType } from '../components/Pagination';
@ -25,30 +24,8 @@ import loadVideoListByFilter from '../api/loader/loadVideoListByPage';
import useIsAdmin from '../functions/useIsAdmin';
import { useUserConfigStore } from '../stores/UserConfigStore';
export type PlaylistType = {
playlist_active: boolean;
playlist_channel: string;
playlist_channel_id: string;
playlist_description: string;
playlist_entries: PlaylistEntryType[];
playlist_id: string;
playlist_last_refresh: string;
playlist_name: string;
playlist_subscribed: boolean;
playlist_thumbnail: string;
playlist_type: string;
_index: string;
_score: number;
};
export type PlaylistResponseType = {
data?: PlaylistType;
config?: ConfigType;
};
export type VideoResponseType = {
data?: VideoType[];
config?: ConfigType;
paginate?: PaginationType;
};
@ -71,19 +48,19 @@ const Playlist = () => {
const [channelResponse, setChannelResponse] = useState<ChannelResponseType>();
const [videoResponse, setVideoResponse] = useState<VideoResponseType>();
const playlist = playlistResponse?.data;
const channel = channelResponse?.data;
const playlist = playlistResponse;
const channel = channelResponse;
const videos = videoResponse?.data;
const pagination = videoResponse?.paginate;
const palylistEntries = playlistResponse?.data?.playlist_entries;
const palylistEntries = playlistResponse?.playlist_entries;
const videoArchivedCount = Number(palylistEntries?.filter(video => video.downloaded).length);
const videoInPlaylistCount = pagination?.total_hits;
const showEmbeddedVideo = videoId !== null;
const view = userConfig.config.view_style_home;
const gridItems = userConfig.config.grid_items;
const hideWatched = userConfig.config.hide_watched;
const view = userConfig.view_style_home;
const gridItems = userConfig.grid_items;
const hideWatched = userConfig.hide_watched;
const isGridView = view === ViewStyles.grid;
const gridView = isGridView ? `boxed-${gridItems}` : '';
const gridViewGrid = isGridView ? `grid-${gridItems}` : '';
@ -98,9 +75,9 @@ const Playlist = () => {
sort: 'downloaded', // downloaded or published? or playlist sort order?
});
const isCustomPlaylist = playlist?.data?.playlist_type === 'custom';
const isCustomPlaylist = playlist?.playlist_type === 'custom';
if (!isCustomPlaylist) {
const channel = await loadChannelById(playlist.data.playlist_channel_id);
const channel = await loadChannelById(playlist.playlist_channel_id);
setChannelResponse(channel);
}
@ -112,7 +89,7 @@ const Playlist = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [
playlistId,
userConfig.config.hide_watched,
userConfig.hide_watched,
refresh,
currentPage,
pagination?.current_page,
@ -298,7 +275,7 @@ const Playlist = () => {
</div>
</div>
{playlist.playlist_description && (
{playlist.playlist_description !== 'False' && (
<div className="description-box">
<p
id={descriptionExpanded ? 'text-expand-expanded' : 'text-expand'}

32
frontend/src/pages/Playlists.tsx
View File

@ -7,10 +7,8 @@ import iconListView from '/img/icon-listview.svg';
import { OutletContextType } from './Base';
import loadPlaylistList from '../api/loader/loadPlaylistList';
import { ConfigType } from './Home';
import Pagination, { PaginationType } from '../components/Pagination';
import PlaylistList from '../components/PlaylistList';
import { PlaylistType } from './Playlist';
import updateBulkPlaylistSubscriptions from '../api/actions/updateBulkPlaylistSubscriptions';
import createCustomPlaylist from '../api/actions/createCustomPlaylist';
import ScrollToTopOnNavigate from '../components/ScrollToTop';
@ -18,23 +16,16 @@ import Button from '../components/Button';
import useIsAdmin from '../functions/useIsAdmin';
import { useUserConfigStore } from '../stores/UserConfigStore';
import Notifications from '../components/Notifications';
export type PlaylistEntryType = {
youtube_id: string;
title: string;
uploader: string;
idx: number;
downloaded: boolean;
};
import updateUserConfig, { UserConfigType } from '../api/actions/updateUserConfig';
import { PlaylistType } from '../api/loader/loadPlaylistById';
export type PlaylistsResponseType = {
data?: PlaylistType[];
config?: ConfigType;
paginate?: PaginationType;
};
const Playlists = () => {
const { userConfig, setPartialConfig } = useUserConfigStore();
const { userConfig, setUserConfig } = useUserConfigStore();
const { currentPage, setCurrentPage } = useOutletContext() as OutletContextType;
const isAdmin = useIsAdmin();
@ -51,8 +42,8 @@ const Playlists = () => {
const hasPlaylists = playlistResponse?.data?.length !== 0;
const view = userConfig.config.view_style_playlist;