-
- - tube-archivist-banner - -
+ +
+
diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css index 5b15bf1..4856105 100644 --- a/tubearchivist/static/css/style.css +++ b/tubearchivist/static/css/style.css @@ -159,13 +159,13 @@ button:hover { } .top-banner { - text-align: center; -} - -.top-banner img { - width: 100%; - max-width: 700px; - content: var(--banner); + background-image: var(--banner); + background-repeat: no-repeat; + background-size: contain; + height: 10vh; + min-height: 80px; + max-height: 120px; + background-position: center center; } .footer { From e75ffb603ccc710c8e191450991f5efb5f4c98f4 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 16 Dec 2023 13:13:48 +0700 Subject: [PATCH 5/8] fix auto delete lte datatype, #622 --- tubearchivist/home/src/download/yt_dlp_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index efcdb9b..6bf2c95 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -50,7 +50,7 @@ class DownloadPostProcess: return print(f"auto delete older than {autodelete_days} days") - now_lte = self.now - autodelete_days * 24 * 60 * 60 + now_lte = str(self.now - autodelete_days * 24 * 60 * 60) data = { "query": {"range": {"player.watched_date": {"lte": now_lte}}}, "sort": [{"player.watched_date": {"order": "asc"}}], @@ -63,7 +63,7 @@ class DownloadPostProcess: if "autodelete_days" in value: autodelete_days = value.get("autodelete_days") print(f"{channel_id}: delete older than {autodelete_days}d") - now_lte = self.now - autodelete_days * 24 * 60 * 60 + now_lte = str(self.now - autodelete_days * 24 * 60 * 60) must_list = [ {"range": {"player.watched_date": {"lte": now_lte}}}, {"term": {"channel.channel_id": {"value": channel_id}}}, From 8870782a6e927fe6a8f6e055a62e664bbe58939a Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 16 Dec 2023 17:39:09 +0700 Subject: [PATCH 6/8] refactor, use decode_responses in base class --- tubearchivist/home/src/ta/ta_redis.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index de821fb..b492aaf 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -20,6 +20,7 @@ class RedisBase: self.conn = redis.Redis( host=EnvironmentSettings.REDIS_HOST, port=EnvironmentSettings.REDIS_PORT, + decode_responses=True, ) @@ -82,7 +83,7 @@ class RedisArchivist(RedisBase): if not reply: return [] - return [i.decode().lstrip(self.NAME_SPACE) for i in reply] + return [i.lstrip(self.NAME_SPACE) for i in reply] def list_items(self, query: str) -> list: """list all matches""" @@ -108,8 +109,7 @@ class RedisQueue(RedisBase): def get_all(self): """return all elements in list""" result = self.conn.execute_command("LRANGE", self.key, 0, -1) - all_elements = [i.decode() for i in result] - return all_elements + return result def length(self) -> int: """return total elements in list""" @@ -136,11 +136,8 @@ class RedisQueue(RedisBase): def get_next(self) -> str | bool: """return next element in the queue, False if none""" result = self.conn.execute_command("LPOP", self.key) - if not result: - return False - next_element = result.decode() - return next_element + return result def clear(self) -> None: """delete list from redis""" @@ -170,7 +167,7 @@ class TaskRedis(RedisBase): def get_all(self) -> list: """return all tasks""" all_keys = self.conn.execute_command("KEYS", f"{self.BASE}*") - return [i.decode().replace(self.BASE, "") for i in all_keys] + return [i.replace(self.BASE, "") for i in all_keys] def get_single(self, task_id: str) -> dict: """return content of single task""" @@ -178,7 +175,7 @@ class TaskRedis(RedisBase): if not result: return {} - return json.loads(result.decode()) + return json.loads(result) def set_key( self, task_id: str, message: dict, expire: bool | int = False From 789c35e2b543e62dbeb481421583fa7bdd49bd69 Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Dec 2023 11:22:26 +0700 Subject: [PATCH 7/8] refactor RedisQueue to use sorted set --- .../config/management/commands/ta_startup.py | 21 ++++---- tubearchivist/home/src/index/reindex.py | 2 +- tubearchivist/home/src/ta/ta_redis.py | 48 +++++++------------ 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 75a5d02..a4e1169 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -36,7 +36,7 @@ class Command(BaseCommand): self.stdout.write(TOPIC) self._sync_redis_state() self._make_folders() - self._release_locks() + self._clear_redis_keys() self._clear_tasks() self._clear_dl_cache() self._version_check() @@ -73,10 +73,10 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS(" ✓ expected folders created")) - def _release_locks(self): - """make sure there are no leftover locks set in redis""" - self.stdout.write("[3] clear leftover locks in redis") - all_locks = [ + def _clear_redis_keys(self): + """make sure there are no leftover locks or keys set in redis""" + self.stdout.write("[3] clear leftover keys in redis") + all_keys = [ "dl_queue_id", "dl_queue", "downloading", @@ -85,19 +85,22 @@ class Command(BaseCommand): "rescan", "run_backup", "startup_check", + "reindex:ta_video", + "reindex:ta_channel", + "reindex:ta_playlist", ] redis_con = RedisArchivist() has_changed = False - for lock in all_locks: - if redis_con.del_message(lock): + for key in all_keys: + if redis_con.del_message(key): self.stdout.write( - self.style.SUCCESS(f" ✓ cleared lock {lock}") + self.style.SUCCESS(f" ✓ cleared key {key}") ) has_changed = True if not has_changed: - self.stdout.write(self.style.SUCCESS(" no locks found")) + self.stdout.write(self.style.SUCCESS(" no keys found")) def _clear_tasks(self): """clear tasks and messages""" diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index 53b62d1..224e542 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -243,7 +243,7 @@ class Reindex(ReindexBase): return for name, index_config in self.REINDEX_CONFIG.items(): - if not RedisQueue(index_config["queue_name"]).has_item(): + if not RedisQueue(index_config["queue_name"]).length(): continue self.total = RedisQueue(index_config["queue_name"]).length() diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index b492aaf..dd9ac51 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -100,61 +100,49 @@ class RedisArchivist(RedisBase): class RedisQueue(RedisBase): - """dynamically interact with queues in redis""" + """ + dynamically interact with queues in redis using sorted set + - low score number is first in queue + - add new items with high score number + """ def __init__(self, queue_name: str): super().__init__() self.key = f"{self.NAME_SPACE}{queue_name}" - def get_all(self): + def get_all(self) -> list[str]: """return all elements in list""" - result = self.conn.execute_command("LRANGE", self.key, 0, -1) + result = self.conn.zrange(self.key, 0, -1) return result def length(self) -> int: """return total elements in list""" - return self.conn.execute_command("LLEN", self.key) + return self.conn.zcard(self.key) def in_queue(self, element) -> str | bool: """check if element is in list""" - result = self.conn.execute_command("LPOS", self.key, element) + result = self.conn.zrank(self.key, element) if result is not None: return "in_queue" return False - def add_list(self, to_add): + def add_list(self, to_add: list) -> None: """add list to queue""" - self.conn.execute_command("RPUSH", self.key, *to_add) - - def add_priority(self, to_add: str) -> None: - """add single video to front of queue""" - item: str = json.dumps(to_add) - self.clear_item(item) - self.conn.execute_command("LPUSH", self.key, item) + mapping = {i: "+inf" for i in to_add} + self.conn.zadd(self.key, mapping) def get_next(self) -> str | bool: - """return next element in the queue, False if none""" - result = self.conn.execute_command("LPOP", self.key) + """return next element in the queue, if available""" + result = self.conn.zpopmin(self.key) + if not result: + return False - return result + return result[0][0] def clear(self) -> None: """delete list from redis""" - self.conn.execute_command("DEL", self.key) - - def clear_item(self, to_clear: str) -> None: - """remove single item from list if it's there""" - self.conn.execute_command("LREM", self.key, 0, to_clear) - - def trim(self, size: int) -> None: - """trim the queue based on settings amount""" - self.conn.execute_command("LTRIM", self.key, 0, size) - - def has_item(self) -> bool: - """check if queue as at least one pending item""" - result = self.conn.execute_command("LRANGE", self.key, 0, 0) - return bool(result) + self.conn.delete(self.key) class TaskRedis(RedisBase): From 7eec3ece4933cf6310ecb6ed63cc1e864d67edcd Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 17 Dec 2023 11:28:36 +0700 Subject: [PATCH 8/8] lock yt-dlp at commit --- tubearchivist/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt index fb3e37a..4b93d0d 100644 --- a/tubearchivist/requirements.txt +++ b/tubearchivist/requirements.txt @@ -10,4 +10,4 @@ requests==2.31.0 ryd-client==0.0.6 uWSGI==2.0.23 whitenoise==6.6.0 -yt-dlp==2023.11.16 +yt-dlp @ git+https://github.com/yt-dlp/yt-dlp@6b5d93b0b0240e287389d1d43b2d5293e18aa4cc