From 85b56300b3125cf351ee60af585b0db36c5ccd55 Mon Sep 17 00:00:00 2001 From: Clark <104835586+anonamouslyginger@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:46:55 +0000 Subject: [PATCH 1/5] Move user configuration from Redis to ES (#533) * ES Client must bootstrap itself to be the source of config If this is not done a cyclic loop is created between the config loader and the ES client. This lays the ground work for ES being the source of all app config. * auto_download is not used anymore * Add UserConfig class that encapsulates user config storage This class will allow the rest of the code to 'not care' about how user properties are stored. This requires the addition of a ta_users index in ES. * Create migration task for user config transfer * Replace getters and setters for each property Strongly type the user configuration Migrate missed sponsorblock ID * Other DB settings will be another PR --- .../config/management/commands/ta_startup.py | 98 +++++++++++++ tubearchivist/home/config.json | 14 -- .../home/src/download/yt_dlp_handler.py | 2 +- tubearchivist/home/src/es/connect.py | 22 +-- tubearchivist/home/src/es/index_mapping.json | 13 +- tubearchivist/home/src/frontend/api_calls.py | 44 +++--- tubearchivist/home/src/frontend/searching.py | 10 +- tubearchivist/home/src/index/generic.py | 10 +- tubearchivist/home/src/index/video.py | 23 ++-- tubearchivist/home/src/ta/config.py | 46 +------ tubearchivist/home/src/ta/users.py | 104 ++++++++++++++ .../home/templates/home/settings_user.html | 4 +- tubearchivist/home/views.py | 129 ++++++------------ 13 files changed, 302 insertions(+), 217 deletions(-) create mode 100644 tubearchivist/home/src/ta/users.py diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index e45c46c..f715b8f 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -16,6 +16,7 @@ from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import clear_dl_cache from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager +from home.src.ta.users import UserConfig TOPIC = """ @@ -44,6 +45,7 @@ class Command(BaseCommand): self._mig_snapshot_check() self._mig_set_streams() self._mig_set_autostart() + self._mig_move_users_to_es() def _sync_redis_state(self): """make sure redis gets new config.json values""" @@ -219,3 +221,99 @@ class Command(BaseCommand): self.stdout.write(response) sleep(60) raise CommandError(message) + + def _mig_move_users_to_es(self): # noqa: C901 + """migration: update from 0.4.1 to 0.5.0 move user config to ES""" + self.stdout.write("[MIGRATION] move user configuration to ES") + redis = RedisArchivist() + + # 1: Find all users in Redis + users = {i.split(":")[0] for i in redis.list_keys("[0-9]*:")} + if not users: + self.stdout.write(" no users needed migrating to ES") + return + + # 2: Write all Redis user settings to ES + # 3: Remove user settings from Redis + try: + for user in users: + new_conf = UserConfig(user) + + colors_key = f"{user}:colors" + colors = redis.get_message(colors_key).get("status") + if colors: + new_conf.set_value("colors", colors) + redis.del_message(colors_key) + + sort_by_key = f"{user}:sort_by" + sort_by = redis.get_message(sort_by_key).get("status") + if sort_by: + new_conf.set_value("sort_by", sort_by) + redis.del_message(sort_by_key) + + page_size_key = f"{user}:page_size" + page_size = redis.get_message(page_size_key).get("status") + if page_size: + new_conf.set_value("page_size", page_size) + redis.del_message(page_size_key) + + sort_order_key = f"{user}:sort_order" + sort_order = redis.get_message(sort_order_key).get("status") + if sort_order: + new_conf.set_value("sort_order", sort_order) + redis.del_message(sort_order_key) + + grid_items_key = f"{user}:grid_items" + grid_items = redis.get_message(grid_items_key).get("status") + if grid_items: + new_conf.set_value("grid_items", grid_items) + redis.del_message(grid_items_key) + + hide_watch_key = f"{user}:hide_watched" + hide_watch = redis.get_message(hide_watch_key).get("status") + if hide_watch: + new_conf.set_value("hide_watched", hide_watch) + redis.del_message(hide_watch_key) + + ignore_only_key = f"{user}:show_ignored_only" + ignore_only = redis.get_message(ignore_only_key).get("status") + if ignore_only: + new_conf.set_value("show_ignored_only", ignore_only) + redis.del_message(ignore_only_key) + + subed_only_key = f"{user}:show_subed_only" + subed_only = redis.get_message(subed_only_key).get("status") + if subed_only: + new_conf.set_value("show_subed_only", subed_only) + redis.del_message(subed_only_key) + + sb_id_key = f"{user}:id_sb_id" + sb_id = redis.get_message(sb_id_key).get("status") + if sb_id: + new_conf.set_value("sb_id_id", sb_id) + redis.del_message(sb_id_key) + + for view in ["channel", "playlist", "home", "downloads"]: + view_key = f"{user}:view:{view}" + view_style = redis.get_message(view_key).get("status") + if view_style: + new_conf.set_value(f"view_style_{view}", view_style) + redis.del_message(view_key) + + self.stdout.write( + self.style.SUCCESS( + f" ✓ Settings for user '{user}' migrated to ES" + ) + ) + except Exception as e: + message = " 🗙 user migration to ES failed" + self.stdout.write(self.style.ERROR(message)) + self.stdout.write(self.style.ERROR(e)) + sleep(60) + raise CommandError(message) + else: + self.stdout.write( + self.style.SUCCESS( + " ✓ Settings for all users migrated to ES" + ) + ) diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index d45a2a1..26d3bf9 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -1,18 +1,5 @@ { - "archive": { - "sort_by": "published", - "sort_order": "desc", - "page_size": 12 - }, - "default_view": { - "home": "grid", - "channel": "list", - "downloads": "list", - "playlist": "grid", - "grid_items": 3 - }, "subscriptions": { - "auto_download": false, "channel_size": 50, "live_channel_size": 50, "shorts_channel_size": 50, @@ -41,7 +28,6 @@ "app_root": "/app", "cache_dir": "/cache", "videos": "/youtube", - "colors": "dark", "enable_cast": false, "enable_snapshot": true }, diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index f2bf800..9a865ab 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -417,7 +417,7 @@ class VideoDownloader: "lang": "painless", }, } - response, _ = ElasticWrap(path, config=self.config).post(data=data) + response, _ = ElasticWrap(path).post(data=data) updated = response.get("updated") if updated: print(f"[download] reset auto start on {updated} videos.") diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index 0b9d554..b526cf4 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -6,9 +6,9 @@ functionality: # pylint: disable=missing-timeout import json +import os import requests -from home.src.ta.config import AppConfig class ElasticWrap: @@ -16,21 +16,13 @@ class ElasticWrap: returns response json and status code tuple """ - def __init__(self, path, config=False): - self.url = False - self.auth = False - self.path = path - self.config = config - self._get_config() + ES_URL: str = str(os.environ.get("ES_URL")) + ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) + ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic") - def _get_config(self): - """add config if not passed""" - if not self.config: - self.config = AppConfig().config - - es_url = self.config["application"]["es_url"] - self.auth = self.config["application"]["es_auth"] - self.url = f"{es_url}/{self.path}" + def __init__(self, path): + self.url = f"{self.ES_URL}/{path}" + self.auth = (self.ES_USER, self.ES_PASS) def get(self, data=False, timeout=10, print_error=True): """get data from es""" diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index 06bf13c..a25b300 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -1,5 +1,16 @@ { "index_config": [{ + "index_name": "config", + "expected_map": { + "config": { + "type": "object" + } + }, + "expected_set": { + "number_of_replicas": "0" + } + }, + { "index_name": "channel", "expected_map": { "channel_id": { @@ -601,4 +612,4 @@ } } ] -} \ No newline at end of file +} diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index 60764ea..c5402ab 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -4,7 +4,7 @@ Functionality: - called via user input """ -from home.src.ta.ta_redis import RedisArchivist +from home.src.ta.users import UserConfig from home.tasks import run_restore_backup @@ -41,10 +41,8 @@ class PostData: def _change_view(self): """process view changes in home, channel, and downloads""" - origin, new_view = self.exec_val.split(":") - key = f"{self.current_user}:view:{origin}" - print(f"change view: {key} to {new_view}") - RedisArchivist().set_message(key, {"status": new_view}) + view, setting = self.exec_val.split(":") + UserConfig(self.current_user).set_value(f"view_style_{view}", setting) return {"success": True} def _change_grid(self): @@ -52,48 +50,38 @@ class PostData: grid_items = int(self.exec_val) grid_items = max(grid_items, 3) grid_items = min(grid_items, 7) - - key = f"{self.current_user}:grid_items" - print(f"change grid items: {grid_items}") - RedisArchivist().set_message(key, {"status": grid_items}) + UserConfig(self.current_user).set_value("grid_items", grid_items) return {"success": True} def _sort_order(self): """change the sort between published to downloaded""" - sort_order = {"status": self.exec_val} if self.exec_val in ["asc", "desc"]: - RedisArchivist().set_message( - f"{self.current_user}:sort_order", sort_order + UserConfig(self.current_user).set_value( + "sort_order", self.exec_val ) else: - RedisArchivist().set_message( - f"{self.current_user}:sort_by", sort_order - ) + UserConfig(self.current_user).set_value("sort_by", self.exec_val) return {"success": True} def _hide_watched(self): """toggle if to show watched vids or not""" - key = f"{self.current_user}:hide_watched" - message = {"status": bool(int(self.exec_val))} - print(f"toggle {key}: {message}") - RedisArchivist().set_message(key, message) + UserConfig(self.current_user).set_value( + "hide_watched", bool(int(self.exec_val)) + ) return {"success": True} def _show_subed_only(self): """show or hide subscribed channels only on channels page""" - key = f"{self.current_user}:show_subed_only" - message = {"status": bool(int(self.exec_val))} - print(f"toggle {key}: {message}") - RedisArchivist().set_message(key, message) + UserConfig(self.current_user).set_value( + "show_subed_only", bool(int(self.exec_val)) + ) return {"success": True} def _show_ignored_only(self): """switch view on /downloads/ to show ignored only""" - show_value = self.exec_val - key = f"{self.current_user}:show_ignored_only" - value = {"status": show_value} - print(f"Filter download view ignored only: {show_value}") - RedisArchivist().set_message(key, value) + UserConfig(self.current_user).set_value( + "show_ignored_only", bool(int(self.exec_val)) + ) return {"success": True} def _db_restore(self): diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/home/src/frontend/searching.py index b9f2624..a150106 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/home/src/frontend/searching.py @@ -11,23 +11,21 @@ from datetime import datetime from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap -from home.src.ta.config import AppConfig from home.src.ta.helper import get_duration_str class SearchHandler: """search elastic search""" - def __init__(self, path, config, data=False): + def __init__(self, path, data=False): self.max_hits = None self.aggs = None self.path = path - self.config = config self.data = data def get_data(self): """get the data""" - response, _ = ElasticWrap(self.path, config=self.config).get(self.data) + response, _ = ElasticWrap(self.path).get(self.data) if "hits" in response.keys(): self.max_hits = response["hits"]["total"]["value"] @@ -109,12 +107,10 @@ class SearchHandler: class SearchForm: """build query from search form data""" - CONFIG = AppConfig().config - def multi_search(self, search_query): """searching through index""" path, query, query_type = SearchParser(search_query).run() - look_up = SearchHandler(path, config=self.CONFIG, data=query) + look_up = SearchHandler(path, data=query) search_results = look_up.get_data() all_results = self.build_results(search_results) diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 6e82e54..a5f624d 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -8,7 +8,7 @@ import math from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig -from home.src.ta.ta_redis import RedisArchivist +from home.src.ta.users import UserConfig class YouTubeItem: @@ -100,13 +100,7 @@ class Pagination: def get_page_size(self): """get default or user modified page_size""" - key = f"{self.request.user.id}:page_size" - page_size = RedisArchivist().get_message(key)["status"] - if not page_size: - config = AppConfig().config - page_size = config["archive"]["page_size"] - - return page_size + return UserConfig(self.request.user.id).get_value("page_size") def first_guess(self): """build first guess before api call""" diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 626d257..606e32f 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -18,7 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor -from home.src.ta.ta_redis import RedisArchivist +from home.src.ta.users import UserConfig from ryd_client import ryd_client @@ -32,17 +32,16 @@ class SponsorBlock: self.user_agent = f"{settings.TA_UPSTREAM} {settings.TA_VERSION}" self.last_refresh = int(datetime.now().timestamp()) - def get_sb_id(self): - """get sponsorblock userid or generate if needed""" + def get_sb_id(self) -> str: + """get sponsorblock for the userid or generate if needed""" if not self.user_id: - print("missing request user id") - raise ValueError + raise ValueError("missing request user id") - key = f"{self.user_id}:id_sponsorblock" - sb_id = RedisArchivist().get_message(key) - if not sb_id["status"]: - sb_id = {"status": randomizor(32)} - RedisArchivist().set_message(key, sb_id) + user = UserConfig(self.user_id) + sb_id = user.get_value("sponsorblock_id") + if not sb_id: + sb_id = randomizor(32) + user.set_value("sponsorblock_id", sb_id) return sb_id @@ -88,7 +87,7 @@ class SponsorBlock: def post_timestamps(self, youtube_id, start_time, end_time): """post timestamps to api""" - user_id = self.get_sb_id().get("status") + user_id = self.get_sb_id() data = { "videoID": youtube_id, "startTime": start_time, @@ -105,7 +104,7 @@ class SponsorBlock: def vote_on_segment(self, uuid, vote): """send vote on existing segment""" - user_id = self.get_sb_id().get("status") + user_id = self.get_sb_id() data = { "UUID": uuid, "userID": user_id, diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 84fe84a..a32d083 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -17,12 +17,10 @@ from home.src.ta.ta_redis import RedisArchivist class AppConfig: - """handle user settings and application variables""" + """handle application variables""" - def __init__(self, user_id=False): - self.user_id = user_id + def __init__(self): self.config = self.get_config() - self.colors = self.get_colors() def get_config(self): """get config from default file or redis if changed""" @@ -30,12 +28,6 @@ class AppConfig: if not config: config = self.get_config_file() - if self.user_id: - key = f"{self.user_id}:page_size" - page_size = RedisArchivist().get_message(key)["status"] - if page_size: - config["archive"]["page_size"] = page_size - config["application"].update(self.get_config_env()) return config @@ -50,14 +42,12 @@ class AppConfig: @staticmethod def get_config_env(): - """read environment application variables""" - es_pass = os.environ.get("ELASTIC_PASSWORD") - es_user = os.environ.get("ELASTIC_USER", default="elastic") + """read environment application variables. + + Connection to ES is managed in ElasticWrap and the + connection to Redis is managed in RedisArchivist.""" application = { - "REDIS_HOST": os.environ.get("REDIS_HOST"), - "es_url": os.environ.get("ES_URL"), - "es_auth": (es_user, es_pass), "HOST_UID": int(os.environ.get("HOST_UID", False)), "HOST_GID": int(os.environ.get("HOST_GID", False)), "enable_cast": bool(os.environ.get("ENABLE_CAST")), @@ -103,30 +93,6 @@ class AppConfig: RedisArchivist().set_message("config", self.config, save=True) return updated - @staticmethod - def set_user_config(form_post, user_id): - """set values in redis for user settings""" - for key, value in form_post.items(): - if not value: - continue - - message = {"status": value} - redis_key = f"{user_id}:{key}" - RedisArchivist().set_message(redis_key, message, save=True) - - def get_colors(self): - """overwrite config if user has set custom values""" - colors = False - if self.user_id: - col_dict = RedisArchivist().get_message(f"{self.user_id}:colors") - colors = col_dict["status"] - - if not colors: - colors = self.config["application"]["colors"] - - self.config["application"]["colors"] = colors - return colors - @staticmethod def _build_rand_daily(): """build random daily schedule per installation""" diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py new file mode 100644 index 0000000..c337381 --- /dev/null +++ b/tubearchivist/home/src/ta/users.py @@ -0,0 +1,104 @@ +""" +Functionality: +- read and write user config backed by ES +- encapsulate persistence of user properties +""" + +from typing import TypedDict + +from home.src.es.connect import ElasticWrap + + +class UserConfigType(TypedDict, total=False): + """describes the user configuration""" + + colors: str + page_size: int + sort_by: str + sort_order: str + view_style_home: str + view_style_channel: str + view_style_downloads: str + view_style_playlist: str + grid_items: int + hide_watched: bool + show_ignored_only: bool + show_subed_only: bool + sponsorblock_id: str + + +class UserConfig: + """Handle settings for an individual user + + Create getters and setters for usage in the application. + Although tedious it helps prevents everything caring about how properties + are persisted. Plus it allows us to save anytime any value is set. + """ + + _DEFAULT_USER_SETTINGS = UserConfigType( + colors="dark", + page_size=12, + sort_by="published", + sort_order="desc", + view_style_home="grid", + view_style_channel="list", + view_style_downloads="list", + view_style_playlist="grid", + grid_items=3, + hide_watched=False, + show_ignored_only=False, + show_subed_only=False, + sponsorblock_id=None, + ) + + def __init__(self, user_id: str): + self._user_id: str = user_id + self._config: UserConfigType = self._get_config() + + def get_value(self, key: str): + """Get the given key from the users configuration + + Throws a KeyError if the requested Key is not a permitted value""" + if key not in self._DEFAULT_USER_SETTINGS: + raise KeyError(f"Unable to read config for unknown key '{key}'") + + return self._config.get(key) or self._DEFAULT_USER_SETTINGS.get(key) + + def set_value(self, key: str, value: str | bool | int): + """Set or replace a configuration value for the user + + Throws a KeyError if the requested Key is not a permitted value""" + if not self._user_id: + raise ValueError("Unable to persist config for null user_id") + + if key not in self._DEFAULT_USER_SETTINGS: + raise KeyError(f"Unable to persist config for unknown key '{key}'") + + old = self.get_value(key) + self._config[key] = value + + # Upsert this property (creating a record if not exists) + es_payload = {"doc": {"config": {key: value}}, "doc_as_upsert": True} + es_document_path = f"ta_config/_update/user_{self._user_id}" + response, status = ElasticWrap(es_document_path).post(es_payload) + if status < 200 or status > 299: + raise ValueError(f"Failed storing user value {status}: {response}") + + print(f"User {self._user_id} value '{key}' change: {old} > {value}") + + def _get_config(self) -> UserConfigType: + """get config from ES or load from the application defaults""" + if not self._user_id: + # this is for a non logged-in user so use all the defaults + return {} + + # Does this user have configuration stored in ES + es_document_path = f"ta_config/_doc/user_{self._user_id}" + response, status = ElasticWrap(es_document_path).get(print_error=False) + if status == 200 and "_source" in response.keys(): + source = response.get("_source") + if "config" in source.keys(): + return source.get("config") + + # There is no config in ES + return {} diff --git a/tubearchivist/home/templates/home/settings_user.html b/tubearchivist/home/templates/home/settings_user.html index a12fc75..b9545e0 100644 --- a/tubearchivist/home/templates/home/settings_user.html +++ b/tubearchivist/home/templates/home/settings_user.html @@ -9,7 +9,7 @@

Color scheme

-

Current color scheme: {{ config.application.colors }}

+

Current color scheme: {{ colors }}

Select your preferred color scheme between dark and light mode.
{{ user_form.colors }}
@@ -17,7 +17,7 @@

Archive View

-

Current page size: {{ config.archive.page_size }}

+

Current page size: {{ page_size }}

Result of videos showing in archive page
{{ user_form.page_size }}
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 24c10fa..b0aea5e 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -41,6 +41,7 @@ from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.helper import time_parser from home.src.ta.ta_redis import RedisArchivist +from home.src.ta.users import UserConfig from home.tasks import index_channel_playlists, subscribe_to from rest_framework.authtoken.models import Token @@ -52,93 +53,38 @@ class ArchivistViewConfig(View): super().__init__() self.view_origin = view_origin self.user_id = False - self.user_conf = False + self.user_conf: UserConfig = False self.default_conf = False self.context = False - def _get_sort_by(self): - """return sort_by config var""" - messag_key = f"{self.user_id}:sort_by" - sort_by = self.user_conf.get_message(messag_key)["status"] - if not sort_by: - sort_by = self.default_conf["archive"]["sort_by"] - - return sort_by - - def _get_sort_order(self): - """return sort_order config var""" - sort_order_key = f"{self.user_id}:sort_order" - sort_order = self.user_conf.get_message(sort_order_key)["status"] - if not sort_order: - sort_order = self.default_conf["archive"]["sort_order"] - - return sort_order - - def _get_view_style(self): - """return view_style config var""" - view_key = f"{self.user_id}:view:{self.view_origin}" - view_style = self.user_conf.get_message(view_key)["status"] - if not view_style: - view_style = self.default_conf["default_view"][self.view_origin] - - return view_style - - def _get_grid_items(self): - """return items per row to show in grid view""" - grid_key = f"{self.user_id}:grid_items" - grid_items = self.user_conf.get_message(grid_key)["status"] - if not grid_items: - grid_items = self.default_conf["default_view"]["grid_items"] - - return grid_items - def get_all_view_styles(self): - """get dict of all view stiles for search form""" - all_keys = ["channel", "playlist", "home"] + """get dict of all view styles for search form""" all_styles = {} - for view_origin in all_keys: - view_key = f"{self.user_id}:view:{view_origin}" - view_style = self.user_conf.get_message(view_key)["status"] - if not view_style: - view_style = self.default_conf["default_view"][view_origin] - all_styles[view_origin] = view_style + for view_origin in ["channel", "playlist", "home", "downloads"]: + all_styles[view_origin] = self.user_conf.get_value( + f"view_style_{view_origin}" + ) return all_styles - def _get_hide_watched(self): - hide_watched_key = f"{self.user_id}:hide_watched" - hide_watched = self.user_conf.get_message(hide_watched_key)["status"] - - return hide_watched - - def _get_show_ignore_only(self): - ignored_key = f"{self.user_id}:show_ignored_only" - show_ignored_only = self.user_conf.get_message(ignored_key)["status"] - - return show_ignored_only - - def _get_show_subed_only(self): - sub_only_key = f"{self.user_id}:show_subed_only" - show_subed_only = self.user_conf.get_message(sub_only_key)["status"] - - return show_subed_only - def config_builder(self, user_id): """build default context for every view""" self.user_id = user_id - self.user_conf = RedisArchivist() - self.default_conf = AppConfig(self.user_id).config + self.user_conf = UserConfig(self.user_id) + self.default_conf = AppConfig().config self.context = { - "colors": self.default_conf["application"]["colors"], + "colors": self.user_conf.get_value("colors"), "cast": self.default_conf["application"]["enable_cast"], - "sort_by": self._get_sort_by(), - "sort_order": self._get_sort_order(), - "view_style": self._get_view_style(), - "grid_items": self._get_grid_items(), - "hide_watched": self._get_hide_watched(), - "show_ignored_only": self._get_show_ignore_only(), - "show_subed_only": self._get_show_subed_only(), + "sort_by": self.user_conf.get_value("sort_by"), + "sort_order": self.user_conf.get_value("sort_order"), + "view_style": self.user_conf.get_value( + f"view_style_{self.view_origin}" + ), + "grid_items": self.user_conf.get_value("grid_items"), + "hide_watched": self.user_conf.get_value("hide_watched"), + "show_ignored_only": self.user_conf.get_value("show_ignored_only"), + "show_subed_only": self.user_conf.get_value("show_subed_only"), "version": settings.TA_VERSION, "ta_update": ReleaseVersion().get_update(), } @@ -212,13 +158,11 @@ class ArchivistResultsView(ArchivistViewConfig): """get all videos in progress""" ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results] data = { - "size": self.default_conf["archive"]["page_size"], + "size": UserConfig(self.user_id).get_value("page_size"), "query": {"bool": {"should": ids}}, "sort": [{"published": {"order": "desc"}}], } - search = SearchHandler( - "ta_video/_search", self.default_conf, data=data - ) + search = SearchHandler("ta_video/_search", data=data) videos = search.get_data() if not videos: return False @@ -236,7 +180,7 @@ class ArchivistResultsView(ArchivistViewConfig): def single_lookup(self, es_path): """retrieve a single item from url""" - search = SearchHandler(es_path, config=self.default_conf) + search = SearchHandler(es_path) result = search.get_data()[0]["source"] return result @@ -251,9 +195,7 @@ class ArchivistResultsView(ArchivistViewConfig): def find_results(self): """add results and pagination to context""" - search = SearchHandler( - self.es_search, config=self.default_conf, data=self.data - ) + search = SearchHandler(self.es_search, data=self.data) self.context["results"] = search.get_data() self.pagination_handler.validate(search.max_hits) self.context["max_hits"] = search.max_hits @@ -268,7 +210,7 @@ class MinView(View): def get_min_context(request): """build minimal vars for context""" return { - "colors": AppConfig(request.user.id).colors, + "colors": UserConfig(request.user.id).get_value("colors"), "version": settings.TA_VERSION, "ta_update": ReleaseVersion().get_update(), } @@ -892,8 +834,8 @@ class VideoView(MinView): def get(self, request, video_id): """get single video""" - config_handler = AppConfig(request.user.id) - look_up = SearchHandler(f"ta_video/_doc/{video_id}", config=False) + config_handler = AppConfig() + look_up = SearchHandler(f"ta_video/_doc/{video_id}") video_data = look_up.get_data()[0]["source"] try: rating = video_data["stats"]["average_rating"] @@ -1005,7 +947,9 @@ class SettingsUserView(MinView): context.update( { "title": "User Settings", - "config": AppConfig(request.user.id).config, + "page_size": UserConfig(request.user.id).get_value( + "page_size" + ), "user_form": UserSettingsForm(), } ) @@ -1015,10 +959,17 @@ class SettingsUserView(MinView): def post(self, request): """handle form post to update settings""" user_form = UserSettingsForm(request.POST) + config_handler = UserConfig(request.user.id) if user_form.is_valid(): user_form_post = user_form.cleaned_data - if any(user_form_post.values()): - AppConfig().set_user_config(user_form_post, request.user.id) + if user_form_post.get("colors"): + config_handler.set_value( + "colors", user_form_post.get("colors") + ) + if user_form_post.get("page_size"): + config_handler.set_value( + "page_size", user_form_post.get("page_size") + ) sleep(1) return redirect("settings_user", permanent=True) @@ -1037,7 +988,7 @@ class SettingsApplicationView(MinView): context.update( { "title": "Application Settings", - "config": AppConfig(request.user.id).config, + "config": AppConfig().config, "api_token": self.get_token(request), "app_form": ApplicationSettingsForm(), "snapshots": ElasticSnapshot().get_snapshot_stats(), @@ -1126,7 +1077,7 @@ class SettingsSchedulingView(MinView): context.update( { "title": "Scheduling Settings", - "config": AppConfig(request.user.id).config, + "config": AppConfig().config, "scheduler_form": SchedulerSettingsForm(), } ) From a5b61bfaf6cc577f50dd514e1cd2b5531fe6bf6f Mon Sep 17 00:00:00 2001 From: Joseph Liu Date: Thu, 21 Sep 2023 08:40:42 -0700 Subject: [PATCH 2/5] Add "Mark Unwatched" to channels and playlists (#547) --- tubearchivist/home/templates/home/channel_id.html | 5 ++++- tubearchivist/home/templates/home/playlist_id.html | 5 ++++- tubearchivist/static/script.js | 10 +++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index dcc935c..421392c 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -47,7 +47,10 @@
{% if aggs %}

{{ aggs.total_items.value }} videos | {{ aggs.total_duration.value_str }} playback | Total size {{ aggs.total_size.value|filesizeformat }}

- +
+ + +
{% endif %}
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index d5d20dd..525e99a 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -50,7 +50,10 @@
{% if max_hits %}

Total Videos archived: {{ max_hits }}/{{ playlist_info.playlist_entries|length }}

-

Watched:

+
+ + +
{% endif %} {% if reindex %}

Reindex scheduled

diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 990e76c..95fe165 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -64,7 +64,15 @@ function isWatchedButton(button) { let youtube_id = button.getAttribute('data-id'); let apiEndpoint = '/api/watched/'; let data = { id: youtube_id, is_watched: true }; - button.remove(); + apiRequest(apiEndpoint, 'POST', data); + setTimeout(function () { + location.reload(); + }, 1000); +} +function isUnwatchedButton(button) { + let youtube_id = button.getAttribute('data-id'); + let apiEndpoint = '/api/watched/'; + let data = { id: youtube_id, is_watched: false }; apiRequest(apiEndpoint, 'POST', data); setTimeout(function () { location.reload(); From 92975a5c95ae460984d7f02298a1bb7431eaa560 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 21 Sep 2023 23:16:06 +0700 Subject: [PATCH 3/5] disable ta_config indexing --- tubearchivist/home/src/es/index_mapping.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json index a25b300..d6dda4b 100644 --- a/tubearchivist/home/src/es/index_mapping.json +++ b/tubearchivist/home/src/es/index_mapping.json @@ -3,7 +3,8 @@ "index_name": "config", "expected_map": { "config": { - "type": "object" + "type": "object", + "enabled": false } }, "expected_set": { From 38b3815a332794f2ba6392aadb3e119be5a99aaa Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 21 Sep 2023 23:17:25 +0700 Subject: [PATCH 4/5] catch disabled old settings --- .../config/management/commands/ta_startup.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index f715b8f..f71fe76 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -241,62 +241,62 @@ class Command(BaseCommand): colors_key = f"{user}:colors" colors = redis.get_message(colors_key).get("status") - if colors: + if colors is not None: new_conf.set_value("colors", colors) redis.del_message(colors_key) sort_by_key = f"{user}:sort_by" sort_by = redis.get_message(sort_by_key).get("status") - if sort_by: + if sort_by is not None: new_conf.set_value("sort_by", sort_by) redis.del_message(sort_by_key) page_size_key = f"{user}:page_size" page_size = redis.get_message(page_size_key).get("status") - if page_size: + if page_size is not None: new_conf.set_value("page_size", page_size) redis.del_message(page_size_key) sort_order_key = f"{user}:sort_order" sort_order = redis.get_message(sort_order_key).get("status") - if sort_order: + if sort_order is not None: new_conf.set_value("sort_order", sort_order) redis.del_message(sort_order_key) grid_items_key = f"{user}:grid_items" grid_items = redis.get_message(grid_items_key).get("status") - if grid_items: + if grid_items is not None: new_conf.set_value("grid_items", grid_items) redis.del_message(grid_items_key) hide_watch_key = f"{user}:hide_watched" hide_watch = redis.get_message(hide_watch_key).get("status") - if hide_watch: + if hide_watch is not None: new_conf.set_value("hide_watched", hide_watch) redis.del_message(hide_watch_key) ignore_only_key = f"{user}:show_ignored_only" ignore_only = redis.get_message(ignore_only_key).get("status") - if ignore_only: + if ignore_only is not None: new_conf.set_value("show_ignored_only", ignore_only) redis.del_message(ignore_only_key) subed_only_key = f"{user}:show_subed_only" subed_only = redis.get_message(subed_only_key).get("status") - if subed_only: + if subed_only is not None: new_conf.set_value("show_subed_only", subed_only) redis.del_message(subed_only_key) sb_id_key = f"{user}:id_sb_id" sb_id = redis.get_message(sb_id_key).get("status") - if sb_id: + if sb_id is not None: new_conf.set_value("sb_id_id", sb_id) redis.del_message(sb_id_key) for view in ["channel", "playlist", "home", "downloads"]: view_key = f"{user}:view:{view}" view_style = redis.get_message(view_key).get("status") - if view_style: + if view_style is not None: new_conf.set_value(f"view_style_{view}", view_style) redis.del_message(view_key) From f423ddc53a1e0ac15b7c8212a1dd79d86a55dd64 Mon Sep 17 00:00:00 2001 From: Igor Rzegocki Date: Fri, 22 Sep 2023 10:01:47 +0200 Subject: [PATCH 5/5] add healthcheck endpoint (#553) --- tubearchivist/config/settings.py | 1 + tubearchivist/home/src/ta/health.py | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tubearchivist/home/src/ta/health.py diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index e58eeea..5629453 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -64,6 +64,7 @@ MIDDLEWARE = [ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "home.src.ta.health.HealthCheckMiddleware", ] ROOT_URLCONF = "config.urls" diff --git a/tubearchivist/home/src/ta/health.py b/tubearchivist/home/src/ta/health.py new file mode 100644 index 0000000..001a021 --- /dev/null +++ b/tubearchivist/home/src/ta/health.py @@ -0,0 +1,11 @@ +from django.http import HttpResponse + + +class HealthCheckMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if request.path == "/health": + return HttpResponse("ok") + return self.get_response(request)