From 285e2042ae57aa46fab461cc07a3d2b93b8481c7 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 14:05:11 +0700 Subject: [PATCH 01/11] [API] add backup endpoints --- tubearchivist/api/urls.py | 10 ++++ tubearchivist/api/views.py | 72 ++++++++++++++++++++++++ tubearchivist/home/src/es/backup.py | 85 +++++++++++++++++------------ tubearchivist/home/tasks.py | 4 +- tubearchivist/static/script.js | 6 +- 5 files changed, 138 insertions(+), 39 deletions(-) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 6ad4163..1f6786a 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -96,6 +96,16 @@ urlpatterns = [ views.SnapshotApiView.as_view(), name="api-snapshot", ), + path( + "backup/", + views.BackupApiListView.as_view(), + name="api-backup-list", + ), + path( + "backup//", + views.BackupApiView.as_view(), + name="api-backup", + ), path( "task-name/", views.TaskListView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index bb72322..3a33f18 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -8,6 +8,7 @@ from home.src.download.subscriptions import ( PlaylistSubscription, ) from home.src.download.yt_dlp_base import CookieHandler +from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.searching import SearchForm @@ -27,6 +28,7 @@ from home.tasks import ( check_reindex, download_pending, extrac_dl, + run_restore_backup, subscribe_to, ) from rest_framework import permissions @@ -764,6 +766,76 @@ class SnapshotApiView(ApiBaseView): return Response(response) +class BackupApiListView(ApiBaseView): + """resolves to /api/backup/ + GET: returns list of available zip backups + POST: take zip backup now + """ + + permission_classes = [AdminOnly] + task_name = "run_backup" + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + backup_files = ElasticBackup().get_all_backup_files() + return Response(backup_files) + + def post(self, request): + """handle post request""" + # pylint: disable=unused-argument + message = TaskCommand().start(self.task_name) + + return Response({"message": message}) + + +class BackupApiView(ApiBaseView): + """resolves to /api/backup// + GET: return a single backup + POST: restore backup + DELETE: delete backup + """ + + permission_classes = [AdminOnly] + task_name = "restore_backup" + + @staticmethod + def get(request, filename): + """get single backup""" + # pylint: disable=unused-argument + backup_file = ElasticBackup().build_backup_file_data(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + return Response(backup_file) + + def post(self, request, filename): + """restore backup file""" + # pylint: disable=unused-argument + task = run_restore_backup.delay(filename) + message = { + "message": "backup restore task started", + "filename": filename, + "task_id": task.id, + } + return Response({"message": message}) + + @staticmethod + def delete(request, filename): + """delete backup file""" + # pylint: disable=unused-argument + + backup_file = ElasticBackup().delete_file(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + message = {"message": f"file {filename} deleted"} + return Response(message) + + class TaskListView(ApiBaseView): """resolves to /api/task-name/ GET: return a list of all stored task results diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index b23592a..1c3778b 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -20,10 +20,11 @@ class ElasticBackup: """dump index to nd-json files for later bulk import""" INDEX_SPLIT = ["comment"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR + BACKUP_DIR = os.path.join(CACHE_DIR, "backup") def __init__(self, reason=False, task=False): self.config = AppConfig().config - self.cache_dir = EnvironmentSettings.CACHE_DIR self.timestamp = datetime.now().strftime("%Y%m%d") self.index_config = get_mapping() self.reason = reason @@ -79,14 +80,13 @@ class ElasticBackup: def zip_it(self): """pack it up into single zip file""" file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip" - folder = os.path.join(self.cache_dir, "backup") to_backup = [] - for file in os.listdir(folder): + for file in os.listdir(self.BACKUP_DIR): if file.endswith(".json"): - to_backup.append(os.path.join(folder, file)) + to_backup.append(os.path.join(self.BACKUP_DIR, file)) - backup_file = os.path.join(folder, file_name) + backup_file = os.path.join(self.BACKUP_DIR, file_name) comp = zipfile.ZIP_DEFLATED with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f: @@ -99,7 +99,7 @@ class ElasticBackup: def post_bulk_restore(self, file_name): """send bulk to es""" - file_path = os.path.join(self.cache_dir, file_name) + file_path = os.path.join(self.CACHE_DIR, file_name) with open(file_path, "r", encoding="utf-8") as f: data = f.read() @@ -110,9 +110,7 @@ class ElasticBackup: def get_all_backup_files(self): """build all available backup files for view""" - backup_dir = os.path.join(self.cache_dir, "backup") - backup_files = os.listdir(backup_dir) - all_backup_files = ignore_filelist(backup_files) + all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR)) all_available_backups = [ i for i in all_backup_files @@ -121,24 +119,36 @@ class ElasticBackup: all_available_backups.sort(reverse=True) backup_dicts = [] - for backup_file in all_available_backups: - file_split = backup_file.split("-") - if len(file_split) == 2: - timestamp = file_split[1].strip(".zip") - reason = False - elif len(file_split) == 3: - timestamp = file_split[1] - reason = file_split[2].strip(".zip") - - to_add = { - "filename": backup_file, - "timestamp": timestamp, - "reason": reason, - } - backup_dicts.append(to_add) + for filename in all_available_backups: + data = self.build_backup_file_data(filename) + backup_dicts.append(data) return backup_dicts + def build_backup_file_data(self, filename): + """build metadata of single backup file""" + file_path = os.path.join(self.BACKUP_DIR, filename) + if not os.path.exists(file_path): + return False + + file_split = filename.split("-") + if len(file_split) == 2: + timestamp = file_split[1].strip(".zip") + reason = False + elif len(file_split) == 3: + timestamp = file_split[1] + reason = file_split[2].strip(".zip") + + data = { + "filename": filename, + "file_path": file_path, + "file_size": os.path.getsize(file_path), + "timestamp": timestamp, + "reason": reason, + } + + return data + def restore(self, filename): """ restore from backup zip file @@ -149,22 +159,19 @@ class ElasticBackup: def _unpack_zip_backup(self, filename): """extract backup zip and return filelist""" - backup_dir = os.path.join(self.cache_dir, "backup") - file_path = os.path.join(backup_dir, filename) + file_path = os.path.join(self.BACKUP_DIR, filename) with zipfile.ZipFile(file_path, "r") as z: zip_content = z.namelist() - z.extractall(backup_dir) + z.extractall(self.BACKUP_DIR) return zip_content def _restore_json_files(self, zip_content): """go through the unpacked files and restore""" - backup_dir = os.path.join(self.cache_dir, "backup") - for idx, json_f in enumerate(zip_content): self._notify_restore(idx, json_f, len(zip_content)) - file_name = os.path.join(backup_dir, json_f) + file_name = os.path.join(self.BACKUP_DIR, json_f) if not json_f.startswith("es_") or not json_f.endswith(".json"): os.remove(file_name) @@ -201,13 +208,21 @@ class ElasticBackup: print("no backup files to rotate") return - backup_dir = os.path.join(self.cache_dir, "backup") - all_to_delete = auto[rotate:] for to_delete in all_to_delete: - file_path = os.path.join(backup_dir, to_delete["filename"]) - print(f"remove old backup file: {file_path}") - os.remove(file_path) + self.delete_file(to_delete["filename"]) + + def delete_file(self, filename): + """delete backup file""" + file_path = os.path.join(self.BACKUP_DIR, filename) + if not os.path.exists(file_path): + print(f"backup file not found: {filename}") + return False + + print(f"remove old backup file: {file_path}") + os.remove(file_path) + + return file_path class BackupCallback: diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 88b0969..e0ab8e7 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -294,7 +294,7 @@ def run_restore_backup(self, filename): if manager.is_pending(self): print(f"[task][{self.name}] restore is already running") self.send_progress("Restore is already running.") - return + return None manager.init(self) self.send_progress(["Reset your Index"]) @@ -302,6 +302,8 @@ def run_restore_backup(self, filename): ElasticBackup(task=self).restore(filename) print("index restore finished") + return f"backup restore completed: {filename}" + @shared_task(bind=True, name="rescan_filesystem", base=BaseTask) def rescan_filesystem(self): diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index cb40b26..07b4f20 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -283,7 +283,7 @@ function reEmbed() { } function dbBackup() { - let apiEndpoint = '/api/task-name/run_backup/'; + let apiEndpoint = '/api/backup/'; apiRequest(apiEndpoint, 'POST'); // clear button let message = document.createElement('p'); @@ -299,8 +299,8 @@ function dbBackup() { function dbRestore(button) { let fileName = button.getAttribute('data-id'); - let payload = JSON.stringify({ 'db-restore': fileName }); - sendPost(payload); + let apiEndpoint = `/api/backup/${fileName}/`; + apiRequest(apiEndpoint, 'POST'); // clear backup row let message = document.createElement('p'); message.innerText = 'restoring from backup'; From 45f4ccfd93ba15c260d96e1037c0c5b71478a6c7 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 14:07:56 +0700 Subject: [PATCH 02/11] fix off by one in filesystem rescan progress --- tubearchivist/home/src/index/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index ab208c2..c650ced 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -83,7 +83,7 @@ class Scanner: if self.task: self.task.send_progress( message_lines=[ - f"Index missing video {youtube_id}, {idx}/{total}" + f"Index missing video {youtube_id}, {idx + 1}/{total}" ], progress=(idx + 1) / total, ) From 31ad9424f523b16e4bc6fa6c449d3ed3dfd6a209 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 14:10:45 +0700 Subject: [PATCH 03/11] remove unused db_restore --- tubearchivist/home/src/frontend/api_calls.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index c5402ab..0d9dc6c 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -5,7 +5,6 @@ Functionality: """ from home.src.ta.users import UserConfig -from home.tasks import run_restore_backup class PostData: @@ -34,7 +33,6 @@ class PostData: "hide_watched": self._hide_watched, "show_subed_only": self._show_subed_only, "show_ignored_only": self._show_ignored_only, - "db-restore": self._db_restore, } return exec_map[self.to_exec] @@ -83,10 +81,3 @@ class PostData: "show_ignored_only", bool(int(self.exec_val)) ) return {"success": True} - - def _db_restore(self): - """restore es zip from settings page""" - print("restoring index from backup zip") - filename = self.exec_val - run_restore_backup.delay(filename) - return {"success": True} From 4b63c2f536cf9aea6b864ec72e9166bcf6b657c0 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 14:33:30 +0700 Subject: [PATCH 04/11] simplify return message --- tubearchivist/api/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 3a33f18..6f2aaad 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -785,9 +785,13 @@ class BackupApiListView(ApiBaseView): def post(self, request): """handle post request""" # pylint: disable=unused-argument - message = TaskCommand().start(self.task_name) + response = TaskCommand().start(self.task_name) + message = { + "message": "backup task started", + "task_id": response["task_id"], + } - return Response({"message": message}) + return Response(message) class BackupApiView(ApiBaseView): @@ -820,7 +824,7 @@ class BackupApiView(ApiBaseView): "filename": filename, "task_id": task.id, } - return Response({"message": message}) + return Response(message) @staticmethod def delete(request, filename): From 4d5aa4ad2fdcb44b7b3a4a42bcb9e9084b85933b Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 17:25:22 +0700 Subject: [PATCH 05/11] validate user config values --- tubearchivist/home/src/ta/users.py | 59 ++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py index 57b342c..f1017fe 100644 --- a/tubearchivist/home/src/ta/users.py +++ b/tubearchivist/home/src/ta/users.py @@ -46,9 +46,15 @@ class UserConfig: sponsorblock_id=None, ) + VALID_COLORS = ["dark", "light"] + VALID_VIEW_STYLE = ["grid", "list"] + VALID_SORT_ORDER = ["asc", "desc"] + VALID_SORT_BY = ["published", "downloaded", "views", "likes"] + VALID_GRID_ITEMS = range(3, 8) + def __init__(self, user_id: str): self._user_id: str = user_id - self._config: UserConfigType = self._get_config() + self._config: UserConfigType = self.get_config() def get_value(self, key: str): """Get the given key from the users configuration @@ -60,15 +66,8 @@ class UserConfig: return self._config.get(key) or self._DEFAULT_USER_SETTINGS.get(key) def set_value(self, key: str, value: str | bool | int): - """Set or replace a configuration value for the user - - Throws a KeyError if the requested Key is not a permitted value""" - if not self._user_id: - raise ValueError("Unable to persist config for null user_id") - - if key not in self._DEFAULT_USER_SETTINGS: - raise KeyError(f"Unable to persist config for unknown key '{key}'") - + """Set or replace a configuration value for the user""" + self._validate(key, value) old = self.get_value(key) self._config[key] = value @@ -79,9 +78,45 @@ class UserConfig: if status < 200 or status > 299: raise ValueError(f"Failed storing user value {status}: {response}") - print(f"User {self._user_id} value '{key}' change: {old} > {value}") + print(f"User {self._user_id} value '{key}' change: {old} -> {value}") - def _get_config(self) -> UserConfigType: + def _validate(self, key, value): + """validate key and value""" + if not self._user_id: + raise ValueError("Unable to persist config for null user_id") + + if key not in self._DEFAULT_USER_SETTINGS: + raise KeyError( + f"Unable to persist config for an unknown key '{key}'" + ) + + valid_values = { + "colors": self.VALID_COLORS, + "sort_by": self.VALID_SORT_BY, + "sort_order": self.VALID_SORT_ORDER, + "view_style_home": self.VALID_VIEW_STYLE, + "view_style_channel": self.VALID_VIEW_STYLE, + "view_style_download": self.VALID_VIEW_STYLE, + "view_style_playlist": self.VALID_VIEW_STYLE, + "grid_items": self.VALID_GRID_ITEMS, + "page_size": int, + "hide_watched": bool, + "show_ignored_only": bool, + "show_subed_only": bool, + } + validation_value = valid_values.get(key) + + if isinstance(validation_value, (list, range)): + if value not in validation_value: + raise ValueError(f"Invalid value for {key}: {value}") + elif validation_value == int: + if not isinstance(value, int): + raise ValueError(f"Invalid value for {key}: {value}") + elif validation_value == bool: + if not isinstance(value, bool): + raise ValueError(f"Invalid value for {key}: {value}") + + def get_config(self) -> UserConfigType: """get config from ES or load from the application defaults""" if not self._user_id: # this is for a non logged-in user so use all the defaults From 0b920e87aeed58b95677bb66985fd9960995fc07 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 19:07:22 +0700 Subject: [PATCH 06/11] [API] add user config endpoints --- tubearchivist/api/urls.py | 5 +++++ tubearchivist/api/views.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 1f6786a..8b075f2 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -121,6 +121,11 @@ urlpatterns = [ views.TaskIDView.as_view(), name="api-task-id", ), + path( + "config/user/", + views.UserConfigView.as_view(), + name="api-config-user", + ), path( "cookie/", views.CookieView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 6f2aaad..0e64d10 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -23,6 +23,7 @@ from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser +from home.src.ta.users import UserConfig from home.tasks import ( BaseTask, check_reindex, @@ -981,6 +982,42 @@ class RefreshView(ApiBaseView): return Response(data) +class UserConfigView(ApiBaseView): + """resolves to /api/config/user/ + GET: return current user config + POST: update user config + """ + + def get(self, request): + """get config""" + user_id = request.user.id + response = UserConfig(user_id).get_config() + response.update({"user_id": user_id}) + + return Response(response) + + def post(self, request): + """update config""" + user_id = request.user.id + data = request.data + + user_conf = UserConfig(user_id) + for key, value in data.items(): + try: + user_conf.set_value(key, value) + except ValueError as err: + message = { + "status": "Bad Request", + "message": f"failed updating {key} to '{value}', {err}", + } + return Response(message, status=400) + + response = user_conf.get_config() + response.update({"user_id": user_id}) + + return Response(response) + + class CookieView(ApiBaseView): """resolves to /api/cookie/ GET: check if cookie is enabled From d677f9579ea4df5be44bf24395b1cd7f3a620cf4 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 1 Nov 2023 22:49:33 +0700 Subject: [PATCH 07/11] replace old process view, use user conf api --- tubearchivist/home/src/frontend/api_calls.py | 83 ------------------- .../home/templates/home/channel_id.html | 4 +- tubearchivist/home/templates/home/home.html | 4 +- tubearchivist/home/urls.py | 1 - tubearchivist/home/views.py | 17 +--- tubearchivist/static/script.js | 40 ++++----- 6 files changed, 23 insertions(+), 126 deletions(-) delete mode 100644 tubearchivist/home/src/frontend/api_calls.py diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py deleted file mode 100644 index 0d9dc6c..0000000 --- a/tubearchivist/home/src/frontend/api_calls.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Functionality: -- collection of functions and tasks from frontend -- called via user input -""" - -from home.src.ta.users import UserConfig - - -class PostData: - """ - map frontend http post values to backend funcs - handover long running tasks to celery - """ - - def __init__(self, post_dict, current_user): - self.post_dict = post_dict - self.to_exec, self.exec_val = list(post_dict.items())[0] - self.current_user = current_user - - def run_task(self): - """execute and return task result""" - to_exec = self.exec_map() - task_result = to_exec() - return task_result - - def exec_map(self): - """map dict key and return function to execute""" - exec_map = { - "change_view": self._change_view, - "change_grid": self._change_grid, - "sort_order": self._sort_order, - "hide_watched": self._hide_watched, - "show_subed_only": self._show_subed_only, - "show_ignored_only": self._show_ignored_only, - } - - return exec_map[self.to_exec] - - def _change_view(self): - """process view changes in home, channel, and downloads""" - view, setting = self.exec_val.split(":") - UserConfig(self.current_user).set_value(f"view_style_{view}", setting) - return {"success": True} - - def _change_grid(self): - """process change items in grid""" - grid_items = int(self.exec_val) - grid_items = max(grid_items, 3) - grid_items = min(grid_items, 7) - UserConfig(self.current_user).set_value("grid_items", grid_items) - return {"success": True} - - def _sort_order(self): - """change the sort between published to downloaded""" - if self.exec_val in ["asc", "desc"]: - UserConfig(self.current_user).set_value( - "sort_order", self.exec_val - ) - else: - UserConfig(self.current_user).set_value("sort_by", self.exec_val) - return {"success": True} - - def _hide_watched(self): - """toggle if to show watched vids or not""" - UserConfig(self.current_user).set_value( - "hide_watched", bool(int(self.exec_val)) - ) - return {"success": True} - - def _show_subed_only(self): - """show or hide subscribed channels only on channels page""" - UserConfig(self.current_user).set_value( - "show_subed_only", bool(int(self.exec_val)) - ) - return {"success": True} - - def _show_ignored_only(self): - """switch view on /downloads/ to show ignored only""" - UserConfig(self.current_user).set_value( - "show_ignored_only", bool(int(self.exec_val)) - ) - return {"success": True} diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 0d45bd8..c51fbd2 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -77,13 +77,13 @@
Sort by: - - diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index 59a3478..0724381 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -60,13 +60,13 @@
Sort by: - - diff --git a/tubearchivist/home/urls.py b/tubearchivist/home/urls.py index 2c2388c..363f505 100644 --- a/tubearchivist/home/urls.py +++ b/tubearchivist/home/urls.py @@ -58,7 +58,6 @@ urlpatterns = [ login_required(views.SettingsActionsView.as_view()), name="settings_actions", ), - path("process/", login_required(views.process), name="process"), path( "channel/", login_required(views.ChannelView.as_view()), diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index dae3d17..bc3a97b 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -4,7 +4,6 @@ Functionality: - holds base classes to inherit from """ import enum -import json import urllib.parse from time import sleep @@ -14,7 +13,7 @@ from django.conf import settings from django.contrib.auth import login from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.forms import AuthenticationForm -from django.http import Http404, JsonResponse +from django.http import Http404 from django.shortcuts import redirect, render from django.utils.decorators import method_decorator from django.views import View @@ -23,7 +22,6 @@ from home.src.download.yt_dlp_base import CookieHandler from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot -from home.src.frontend.api_calls import PostData from home.src.frontend.forms import ( AddToQueueForm, ApplicationSettingsForm, @@ -1133,16 +1131,3 @@ class SettingsActionsView(MinView): ) return render(request, "home/settings_actions.html", context) - - -def process(request): - """handle all the buttons calls via POST ajax""" - if request.method == "POST": - current_user = request.user.id - post_dict = json.loads(request.body.decode()) - post_handler = PostData(post_dict, current_user) - if post_handler.to_exec: - task_result = post_handler.run_task() - return JsonResponse(task_result) - - return JsonResponse({"success": False}) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 07b4f20..0bcfac1 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -2,9 +2,11 @@ /* globals checkMessages */ -function sortChange(sortValue) { - let payload = JSON.stringify({ sort_order: sortValue }); - sendPost(payload); +function sortChange(button) { + let apiEndpoint = '/api/config/user/'; + let data = {}; + data[button.name] = button.value; + apiRequest(apiEndpoint, 'POST', data); setTimeout(function () { location.reload(); }, 500); @@ -105,17 +107,21 @@ function subscribeStatus(subscribeButton) { function changeView(image) { let sourcePage = image.getAttribute('data-origin'); let newView = image.getAttribute('data-value'); - let payload = JSON.stringify({ change_view: sourcePage + ':' + newView }); - sendPost(payload); + let apiEndpoint = '/api/config/user/'; + let data = {}; + data[`view_style_${sourcePage}`] = newView; + console.log(data); + apiRequest(apiEndpoint, 'POST', data); setTimeout(function () { location.reload(); }, 500); } function changeGridItems(image) { - let operator = image.getAttribute('data-value'); - let payload = JSON.stringify({ change_grid: operator }); - sendPost(payload); + let newGridItems = Number(image.getAttribute('data-value')); + let apiEndpoint = '/api/config/user/'; + let data = { grid_items: newGridItems }; + apiRequest(apiEndpoint, 'POST', data); setTimeout(function () { location.reload(); }, 500); @@ -123,12 +129,10 @@ function changeGridItems(image) { function toggleCheckbox(checkbox) { // pass checkbox id as key and checkbox.checked as value - let toggleId = checkbox.id; - let toggleVal = checkbox.checked; - let payloadDict = {}; - payloadDict[toggleId] = toggleVal; - let payload = JSON.stringify(payloadDict); - sendPost(payload); + let apiEndpoint = '/api/config/user/'; + let data = {}; + data[checkbox.id] = checkbox.checked; + apiRequest(apiEndpoint, 'POST', data); setTimeout(function () { let currPage = window.location.pathname; window.location.replace(currPage); @@ -1304,14 +1308,6 @@ function populateEmpty() { // generic -function sendPost(payload) { - let http = new XMLHttpRequest(); - http.open('POST', '/process/', true); - http.setRequestHeader('X-CSRFToken', getCookie('csrftoken')); - http.setRequestHeader('Content-type', 'application/json'); - http.send(payload); -} - function getCookie(c_name) { if (document.cookie.length > 0) { let c_start = document.cookie.indexOf(c_name + '='); From ef6d3e868d4a7f61c54729fbc692c67b8320b0ab Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 8 Nov 2023 23:09:55 +0700 Subject: [PATCH 08/11] bump requirements --- tubearchivist/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index e5a5daa..a36184c 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -1,6 +1,6 @@ apprise==1.6.0 celery==5.3.4 -Django==4.2.6 +Django==4.2.7 django-auth-ldap==4.6.0 django-cors-headers==4.3.0 djangorestframework==3.14.0 @@ -8,6 +8,6 @@ Pillow==10.1.0 redis==5.0.1 requests==2.31.0 ryd-client==0.0.6 -uWSGI==2.0.22 +uWSGI==2.0.23 whitenoise==6.6.0 yt-dlp==2023.10.13 From 1188e66f37262c5ae97407d6d0742e18605224ec Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 8 Nov 2023 23:20:13 +0700 Subject: [PATCH 09/11] fix channel about page parsing, #587 --- tubearchivist/home/src/index/channel.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 02e5726..ada6105 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -23,20 +23,13 @@ class YoutubeChannel(YouTubeItem): es_path = False index_name = "ta_channel" yt_base = "https://www.youtube.com/channel/" - yt_obs = { - "extract_flat": True, - "allow_playlist_files": True, - } + yt_obs = {"playlist_items": "0,0"} def __init__(self, youtube_id, task=False): super().__init__(youtube_id) self.all_playlists = False self.task = task - def build_yt_url(self): - """overwrite base to use channel about page""" - return f"{self.yt_base}{self.youtube_id}/about" - def build_json(self, upload=False, fallback=False): """get from es or from youtube""" self.get_from_es() @@ -69,7 +62,7 @@ class YoutubeChannel(YouTubeItem): "channel_banner_url": self._get_banner_art(), "channel_thumb_url": self._get_thumb_art(), "channel_tvart_url": self._get_tv_art(), - "channel_views": self.youtube_meta.get("view_count", 0), + "channel_views": self.youtube_meta.get("view_count") or 0, } def _parse_tags(self, tags): From 6bc0111d0abd9c4d27ce6980516e84837775fd9a Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 9 Nov 2023 09:31:19 +0700 Subject: [PATCH 10/11] set and get playerVolume from localStorage --- tubearchivist/static/script.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 0bcfac1..84aad94 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -546,7 +546,7 @@ function createVideoTag(videoData, videoProgress) { } let videoTag = ` -