diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index 6ad4163..8b075f2 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -96,6 +96,16 @@ urlpatterns = [ views.SnapshotApiView.as_view(), name="api-snapshot", ), + path( + "backup/", + views.BackupApiListView.as_view(), + name="api-backup-list", + ), + path( + "backup//", + views.BackupApiView.as_view(), + name="api-backup", + ), path( "task-name/", views.TaskListView.as_view(), @@ -111,6 +121,11 @@ urlpatterns = [ views.TaskIDView.as_view(), name="api-task-id", ), + path( + "config/user/", + views.UserConfigView.as_view(), + name="api-config-user", + ), path( "cookie/", views.CookieView.as_view(), diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index bb72322..0e64d10 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -8,6 +8,7 @@ from home.src.download.subscriptions import ( PlaylistSubscription, ) from home.src.download.yt_dlp_base import CookieHandler +from home.src.es.backup import ElasticBackup from home.src.es.connect import ElasticWrap from home.src.es.snapshot import ElasticSnapshot from home.src.frontend.searching import SearchForm @@ -22,11 +23,13 @@ from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser +from home.src.ta.users import UserConfig from home.tasks import ( BaseTask, check_reindex, download_pending, extrac_dl, + run_restore_backup, subscribe_to, ) from rest_framework import permissions @@ -764,6 +767,80 @@ class SnapshotApiView(ApiBaseView): return Response(response) +class BackupApiListView(ApiBaseView): + """resolves to /api/backup/ + GET: returns list of available zip backups + POST: take zip backup now + """ + + permission_classes = [AdminOnly] + task_name = "run_backup" + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + backup_files = ElasticBackup().get_all_backup_files() + return Response(backup_files) + + def post(self, request): + """handle post request""" + # pylint: disable=unused-argument + response = TaskCommand().start(self.task_name) + message = { + "message": "backup task started", + "task_id": response["task_id"], + } + + return Response(message) + + +class BackupApiView(ApiBaseView): + """resolves to /api/backup// + GET: return a single backup + POST: restore backup + DELETE: delete backup + """ + + permission_classes = [AdminOnly] + task_name = "restore_backup" + + @staticmethod + def get(request, filename): + """get single backup""" + # pylint: disable=unused-argument + backup_file = ElasticBackup().build_backup_file_data(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + return Response(backup_file) + + def post(self, request, filename): + """restore backup file""" + # pylint: disable=unused-argument + task = run_restore_backup.delay(filename) + message = { + "message": "backup restore task started", + "filename": filename, + "task_id": task.id, + } + return Response(message) + + @staticmethod + def delete(request, filename): + """delete backup file""" + # pylint: disable=unused-argument + + backup_file = ElasticBackup().delete_file(filename) + if not backup_file: + message = {"message": "file not found"} + return Response(message, status=404) + + message = {"message": f"file {filename} deleted"} + return Response(message) + + class TaskListView(ApiBaseView): """resolves to /api/task-name/ GET: return a list of all stored task results @@ -905,6 +982,42 @@ class RefreshView(ApiBaseView): return Response(data) +class UserConfigView(ApiBaseView): + """resolves to /api/config/user/ + GET: return current user config + POST: update user config + """ + + def get(self, request): + """get config""" + user_id = request.user.id + response = UserConfig(user_id).get_config() + response.update({"user_id": user_id}) + + return Response(response) + + def post(self, request): + """update config""" + user_id = request.user.id + data = request.data + + user_conf = UserConfig(user_id) + for key, value in data.items(): + try: + user_conf.set_value(key, value) + except ValueError as err: + message = { + "status": "Bad Request", + "message": f"failed updating {key} to '{value}', {err}", + } + return Response(message, status=400) + + response = user_conf.get_config() + response.update({"user_id": user_id}) + + return Response(response) + + class CookieView(ApiBaseView): """resolves to /api/cookie/ GET: check if cookie is enabled diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index b10fc99..75a5d02 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -162,11 +162,11 @@ class Command(BaseCommand): for user in users: new_conf = UserConfig(user) - colors_key = f"{user}:colors" - colors = redis.get_message(colors_key).get("status") - if colors: - new_conf.set_value("colors", colors) - redis.del_message(colors_key) + stylesheet_key = f"{user}:color" + stylesheet = redis.get_message(stylesheet_key).get("status") + if stylesheet: + new_conf.set_value("stylesheet", stylesheet) + redis.del_message(stylesheet_key) sort_by_key = f"{user}:sort_by" sort_by = redis.get_message(sort_by_key).get("status") diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index b23592a..1c3778b 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -20,10 +20,11 @@ class ElasticBackup: """dump index to nd-json files for later bulk import""" INDEX_SPLIT = ["comment"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR + BACKUP_DIR = os.path.join(CACHE_DIR, "backup") def __init__(self, reason=False, task=False): self.config = AppConfig().config - self.cache_dir = EnvironmentSettings.CACHE_DIR self.timestamp = datetime.now().strftime("%Y%m%d") self.index_config = get_mapping() self.reason = reason @@ -79,14 +80,13 @@ class ElasticBackup: def zip_it(self): """pack it up into single zip file""" file_name = f"ta_backup-{self.timestamp}-{self.reason}.zip" - folder = os.path.join(self.cache_dir, "backup") to_backup = [] - for file in os.listdir(folder): + for file in os.listdir(self.BACKUP_DIR): if file.endswith(".json"): - to_backup.append(os.path.join(folder, file)) + to_backup.append(os.path.join(self.BACKUP_DIR, file)) - backup_file = os.path.join(folder, file_name) + backup_file = os.path.join(self.BACKUP_DIR, file_name) comp = zipfile.ZIP_DEFLATED with zipfile.ZipFile(backup_file, "w", compression=comp) as zip_f: @@ -99,7 +99,7 @@ class ElasticBackup: def post_bulk_restore(self, file_name): """send bulk to es""" - file_path = os.path.join(self.cache_dir, file_name) + file_path = os.path.join(self.CACHE_DIR, file_name) with open(file_path, "r", encoding="utf-8") as f: data = f.read() @@ -110,9 +110,7 @@ class ElasticBackup: def get_all_backup_files(self): """build all available backup files for view""" - backup_dir = os.path.join(self.cache_dir, "backup") - backup_files = os.listdir(backup_dir) - all_backup_files = ignore_filelist(backup_files) + all_backup_files = ignore_filelist(os.listdir(self.BACKUP_DIR)) all_available_backups = [ i for i in all_backup_files @@ -121,24 +119,36 @@ class ElasticBackup: all_available_backups.sort(reverse=True) backup_dicts = [] - for backup_file in all_available_backups: - file_split = backup_file.split("-") - if len(file_split) == 2: - timestamp = file_split[1].strip(".zip") - reason = False - elif len(file_split) == 3: - timestamp = file_split[1] - reason = file_split[2].strip(".zip") - - to_add = { - "filename": backup_file, - "timestamp": timestamp, - "reason": reason, - } - backup_dicts.append(to_add) + for filename in all_available_backups: + data = self.build_backup_file_data(filename) + backup_dicts.append(data) return backup_dicts + def build_backup_file_data(self, filename): + """build metadata of single backup file""" + file_path = os.path.join(self.BACKUP_DIR, filename) + if not os.path.exists(file_path): + return False + + file_split = filename.split("-") + if len(file_split) == 2: + timestamp = file_split[1].strip(".zip") + reason = False + elif len(file_split) == 3: + timestamp = file_split[1] + reason = file_split[2].strip(".zip") + + data = { + "filename": filename, + "file_path": file_path, + "file_size": os.path.getsize(file_path), + "timestamp": timestamp, + "reason": reason, + } + + return data + def restore(self, filename): """ restore from backup zip file @@ -149,22 +159,19 @@ class ElasticBackup: def _unpack_zip_backup(self, filename): """extract backup zip and return filelist""" - backup_dir = os.path.join(self.cache_dir, "backup") - file_path = os.path.join(backup_dir, filename) + file_path = os.path.join(self.BACKUP_DIR, filename) with zipfile.ZipFile(file_path, "r") as z: zip_content = z.namelist() - z.extractall(backup_dir) + z.extractall(self.BACKUP_DIR) return zip_content def _restore_json_files(self, zip_content): """go through the unpacked files and restore""" - backup_dir = os.path.join(self.cache_dir, "backup") - for idx, json_f in enumerate(zip_content): self._notify_restore(idx, json_f, len(zip_content)) - file_name = os.path.join(backup_dir, json_f) + file_name = os.path.join(self.BACKUP_DIR, json_f) if not json_f.startswith("es_") or not json_f.endswith(".json"): os.remove(file_name) @@ -201,13 +208,21 @@ class ElasticBackup: print("no backup files to rotate") return - backup_dir = os.path.join(self.cache_dir, "backup") - all_to_delete = auto[rotate:] for to_delete in all_to_delete: - file_path = os.path.join(backup_dir, to_delete["filename"]) - print(f"remove old backup file: {file_path}") - os.remove(file_path) + self.delete_file(to_delete["filename"]) + + def delete_file(self, filename): + """delete backup file""" + file_path = os.path.join(self.BACKUP_DIR, filename) + if not os.path.exists(file_path): + print(f"backup file not found: {filename}") + return False + + print(f"remove old backup file: {file_path}") + os.remove(file_path) + + return file_path class BackupCallback: diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py deleted file mode 100644 index c5402ab..0000000 --- a/tubearchivist/home/src/frontend/api_calls.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Functionality: -- collection of functions and tasks from frontend -- called via user input -""" - -from home.src.ta.users import UserConfig -from home.tasks import run_restore_backup - - -class PostData: - """ - map frontend http post values to backend funcs - handover long running tasks to celery - """ - - def __init__(self, post_dict, current_user): - self.post_dict = post_dict - self.to_exec, self.exec_val = list(post_dict.items())[0] - self.current_user = current_user - - def run_task(self): - """execute and return task result""" - to_exec = self.exec_map() - task_result = to_exec() - return task_result - - def exec_map(self): - """map dict key and return function to execute""" - exec_map = { - "change_view": self._change_view, - "change_grid": self._change_grid, - "sort_order": self._sort_order, - "hide_watched": self._hide_watched, - "show_subed_only": self._show_subed_only, - "show_ignored_only": self._show_ignored_only, - "db-restore": self._db_restore, - } - - return exec_map[self.to_exec] - - def _change_view(self): - """process view changes in home, channel, and downloads""" - view, setting = self.exec_val.split(":") - UserConfig(self.current_user).set_value(f"view_style_{view}", setting) - return {"success": True} - - def _change_grid(self): - """process change items in grid""" - grid_items = int(self.exec_val) - grid_items = max(grid_items, 3) - grid_items = min(grid_items, 7) - UserConfig(self.current_user).set_value("grid_items", grid_items) - return {"success": True} - - def _sort_order(self): - """change the sort between published to downloaded""" - if self.exec_val in ["asc", "desc"]: - UserConfig(self.current_user).set_value( - "sort_order", self.exec_val - ) - else: - UserConfig(self.current_user).set_value("sort_by", self.exec_val) - return {"success": True} - - def _hide_watched(self): - """toggle if to show watched vids or not""" - UserConfig(self.current_user).set_value( - "hide_watched", bool(int(self.exec_val)) - ) - return {"success": True} - - def _show_subed_only(self): - """show or hide subscribed channels only on channels page""" - UserConfig(self.current_user).set_value( - "show_subed_only", bool(int(self.exec_val)) - ) - return {"success": True} - - def _show_ignored_only(self): - """switch view on /downloads/ to show ignored only""" - UserConfig(self.current_user).set_value( - "show_ignored_only", bool(int(self.exec_val)) - ) - return {"success": True} - - def _db_restore(self): - """restore es zip from settings page""" - print("restoring index from backup zip") - filename = self.exec_val - run_restore_backup.delay(filename) - return {"success": True} diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index fe5f5b2..5044456 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -2,9 +2,12 @@ - hold all form classes used in the views """ +import os + from django import forms from django.contrib.auth.forms import AuthenticationForm from django.forms.widgets import PasswordInput, TextInput +from home.src.ta.helper import get_stylesheets class CustomAuthForm(AuthenticationForm): @@ -29,14 +32,16 @@ class CustomAuthForm(AuthenticationForm): class UserSettingsForm(forms.Form): """user configurations values""" - CHOICES = [ - ("", "-- change color scheme --"), - ("dark", "Dark"), - ("light", "Light"), - ] + STYLESHEET_CHOICES = [("", "-- change stylesheet --")] + STYLESHEET_CHOICES.extend( + [ + (stylesheet, os.path.splitext(stylesheet)[0].title()) + for stylesheet in get_stylesheets() + ] + ) - colors = forms.ChoiceField( - widget=forms.Select, choices=CHOICES, required=False + stylesheet = forms.ChoiceField( + widget=forms.Select, choices=STYLESHEET_CHOICES, required=False ) page_size = forms.IntegerField(required=False) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 02e5726..ada6105 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -23,20 +23,13 @@ class YoutubeChannel(YouTubeItem): es_path = False index_name = "ta_channel" yt_base = "https://www.youtube.com/channel/" - yt_obs = { - "extract_flat": True, - "allow_playlist_files": True, - } + yt_obs = {"playlist_items": "0,0"} def __init__(self, youtube_id, task=False): super().__init__(youtube_id) self.all_playlists = False self.task = task - def build_yt_url(self): - """overwrite base to use channel about page""" - return f"{self.yt_base}{self.youtube_id}/about" - def build_json(self, upload=False, fallback=False): """get from es or from youtube""" self.get_from_es() @@ -69,7 +62,7 @@ class YoutubeChannel(YouTubeItem): "channel_banner_url": self._get_banner_art(), "channel_thumb_url": self._get_thumb_art(), "channel_tvart_url": self._get_tv_art(), - "channel_views": self.youtube_meta.get("view_count", 0), + "channel_views": self.youtube_meta.get("view_count") or 0, } def _parse_tags(self, tags): diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index ab208c2..c650ced 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -83,7 +83,7 @@ class Scanner: if self.task: self.task.send_progress( message_lines=[ - f"Index missing video {youtube_id}, {idx}/{total}" + f"Index missing video {youtube_id}, {idx + 1}/{total}" ], progress=(idx + 1) / total, ) diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index 8bce809..e3089d8 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -12,6 +12,7 @@ from datetime import datetime from urllib.parse import urlparse import requests +from home.src.ta.settings import EnvironmentSettings def ignore_filelist(filelist: list[str]) -> list[str]: @@ -203,3 +204,21 @@ def ta_host_parser(ta_host: str) -> tuple[list[str], list[str]]: csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}") return allowed_hosts, csrf_trusted_origins + + +def get_stylesheets(): + """Get all valid stylesheets from /static/css""" + app_root = EnvironmentSettings.APP_DIR + stylesheets = os.listdir(os.path.join(app_root, "static/css")) + stylesheets.remove("style.css") + stylesheets.sort() + stylesheets = list(filter(lambda x: x.endswith(".css"), stylesheets)) + return stylesheets + + +def check_stylesheet(stylesheet: str): + """Check if a stylesheet exists. Return dark.css as a fallback""" + if stylesheet in get_stylesheets(): + return stylesheet + + return "dark.css" diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py index 57b342c..bb6a387 100644 --- a/tubearchivist/home/src/ta/users.py +++ b/tubearchivist/home/src/ta/users.py @@ -7,12 +7,13 @@ Functionality: from typing import TypedDict from home.src.es.connect import ElasticWrap +from home.src.ta.helper import get_stylesheets class UserConfigType(TypedDict, total=False): """describes the user configuration""" - colors: str + stylesheet: str page_size: int sort_by: str sort_order: str @@ -31,7 +32,7 @@ class UserConfig: """Handle settings for an individual user""" _DEFAULT_USER_SETTINGS = UserConfigType( - colors="dark", + stylesheet="dark.css", page_size=12, sort_by="published", sort_order="desc", @@ -46,9 +47,15 @@ class UserConfig: sponsorblock_id=None, ) + VALID_STYLESHEETS = get_stylesheets() + VALID_VIEW_STYLE = ["grid", "list"] + VALID_SORT_ORDER = ["asc", "desc"] + VALID_SORT_BY = ["published", "downloaded", "views", "likes"] + VALID_GRID_ITEMS = range(3, 8) + def __init__(self, user_id: str): self._user_id: str = user_id - self._config: UserConfigType = self._get_config() + self._config: UserConfigType = self.get_config() def get_value(self, key: str): """Get the given key from the users configuration @@ -60,15 +67,8 @@ class UserConfig: return self._config.get(key) or self._DEFAULT_USER_SETTINGS.get(key) def set_value(self, key: str, value: str | bool | int): - """Set or replace a configuration value for the user - - Throws a KeyError if the requested Key is not a permitted value""" - if not self._user_id: - raise ValueError("Unable to persist config for null user_id") - - if key not in self._DEFAULT_USER_SETTINGS: - raise KeyError(f"Unable to persist config for unknown key '{key}'") - + """Set or replace a configuration value for the user""" + self._validate(key, value) old = self.get_value(key) self._config[key] = value @@ -79,9 +79,45 @@ class UserConfig: if status < 200 or status > 299: raise ValueError(f"Failed storing user value {status}: {response}") - print(f"User {self._user_id} value '{key}' change: {old} > {value}") + print(f"User {self._user_id} value '{key}' change: {old} -> {value}") - def _get_config(self) -> UserConfigType: + def _validate(self, key, value): + """validate key and value""" + if not self._user_id: + raise ValueError("Unable to persist config for null user_id") + + if key not in self._DEFAULT_USER_SETTINGS: + raise KeyError( + f"Unable to persist config for an unknown key '{key}'" + ) + + valid_values = { + "stylesheet": self.VALID_STYLESHEETS, + "sort_by": self.VALID_SORT_BY, + "sort_order": self.VALID_SORT_ORDER, + "view_style_home": self.VALID_VIEW_STYLE, + "view_style_channel": self.VALID_VIEW_STYLE, + "view_style_download": self.VALID_VIEW_STYLE, + "view_style_playlist": self.VALID_VIEW_STYLE, + "grid_items": self.VALID_GRID_ITEMS, + "page_size": int, + "hide_watched": bool, + "show_ignored_only": bool, + "show_subed_only": bool, + } + validation_value = valid_values.get(key) + + if isinstance(validation_value, (list, range)): + if value not in validation_value: + raise ValueError(f"Invalid value for {key}: {value}") + elif validation_value == int: + if not isinstance(value, int): + raise ValueError(f"Invalid value for {key}: {value}") + elif validation_value == bool: + if not isinstance(value, bool): + raise ValueError(f"Invalid value for {key}: {value}") + + def get_config(self) -> UserConfigType: """get config from ES or load from the application defaults""" if not self._user_id: # this is for a non logged-in user so use all the defaults diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index 88b0969..e0ab8e7 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -294,7 +294,7 @@ def run_restore_backup(self, filename): if manager.is_pending(self): print(f"[task][{self.name}] restore is already running") self.send_progress("Restore is already running.") - return + return None manager.init(self) self.send_progress(["Reset your Index"]) @@ -302,6 +302,8 @@ def run_restore_backup(self, filename): ElasticBackup(task=self).restore(filename) print("index restore finished") + return f"backup restore completed: {filename}" + @shared_task(bind=True, name="rescan_filesystem", base=BaseTask) def rescan_filesystem(self): diff --git a/tubearchivist/home/templates/home/base.html b/tubearchivist/home/templates/home/base.html index b7b998e..40211f2 100644 --- a/tubearchivist/home/templates/home/base.html +++ b/tubearchivist/home/templates/home/base.html @@ -23,11 +23,7 @@ {% else %} TubeArchivist {% endif %} - {% if colors == "dark" %} - - {% else %} - - {% endif %} + {% if cast %} @@ -39,12 +35,7 @@
- {% if colors == 'dark' %} - tube-archivist-banner - {% endif %} - {% if colors == 'light' %} - tube-archivist-banner - {% endif %} + tube-archivist-banner
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 0d45bd8..c51fbd2 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -77,13 +77,13 @@
Sort by: - - diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index 59a3478..0724381 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -60,13 +60,13 @@
Sort by: - - diff --git a/tubearchivist/home/templates/home/login.html b/tubearchivist/home/templates/home/login.html index ecefb42..fe1d24f 100644 --- a/tubearchivist/home/templates/home/login.html +++ b/tubearchivist/home/templates/home/login.html @@ -18,20 +18,11 @@ - {% if colors == "dark" %} - - {% else %} - - {% endif %} +