Finetune notifications, #build

Changed:
- Changed TA_HOST parser, fixing #441
- Changed thumb resync page size
- Added video id to download processing notification
- Added more notifications for zip backup

deploy.sh

@@ -26,6 +26,7 @@ function sync_blackhole {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
     ssh "$host" 'docker build -t bbilly1/tubearchivist --build-arg TARGETPLATFORM="linux/amd64" tubearchivist'
@@ -49,6 +50,7 @@ function sync_test {
         --exclude "**/cache" \
         --exclude "**/__pycache__/" \
         --exclude "db.sqlite3" \
+        --exclude ".mypy_cache" \
         . -e ssh "$host":tubearchivist
 
     # copy default docker-compose file if not exist

tubearchivist/config/settings.py

@@ -18,6 +18,7 @@ import ldap
 from corsheaders.defaults import default_headers
 from django_auth_ldap.config import LDAPSearch
 from home.src.ta.config import AppConfig
+from home.src.ta.helper import ta_host_parser
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -26,23 +27,13 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
-PW_HASH = hashlib.sha256(environ.get("TA_PASSWORD").encode())
+PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
 SECRET_KEY = PW_HASH.hexdigest()
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = bool(environ.get("DJANGO_DEBUG"))
 
-ALLOWED_HOSTS = []
-if environ.get("TA_HOST"):
-    ALLOWED_HOSTS = [i.strip() for i in environ.get("TA_HOST").split()]
-
-CSRF_TRUSTED_ORIGINS = []
-for host in ALLOWED_HOSTS:
-    if host.startswith("http://") or host.startswith("https://"):
-        CSRF_TRUSTED_ORIGINS.append(host)
-    else:
-        CSRF_TRUSTED_ORIGINS.append(f"http://{host}")
-
+ALLOWED_HOSTS, CSRF_TRUSTED_ORIGINS = ta_host_parser(environ["TA_HOST"])
 
 # Application definition
 

tubearchivist/home/src/download/thumbnails.py

@@ -358,6 +358,7 @@ class ThumbFilesystem:
         paginate = IndexPaginate(
             index_name=self.INDEX_NAME,
             data=data,
+            size=200,
             callback=EmbedCallback,
             task=self.task,
             total=self._get_total(),

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -189,7 +189,12 @@ class VideoDownloader:
                 continue
 
             if self.task:
-                self.task.send_progress(["Add video metadata to index."])
+                self.task.send_progress(
+                    [
+                        f"Processing video {youtube_id}",
+                        "Add video metadata to index.",
+                    ]
+                )
 
             vid_dict = index_new_video(
                 youtube_id,
@@ -200,7 +205,12 @@ class VideoDownloader:
             self.videos.add(vid_dict["youtube_id"])
 
             if self.task:
-                self.task.send_progress(["Move downloaded file to archive."])
+                self.task.send_progress(
+                    [
+                        f"Processing video {youtube_id}",
+                        "Move downloaded file to archive.",
+                    ]
+                )
 
             self.move_to_archive(vid_dict)
 

tubearchivist/home/src/es/backup.py

@@ -32,6 +32,7 @@ class ElasticBackup:
         if not self.reason:
             raise ValueError("missing backup reason in ElasticBackup")
 
+        self.task.send_progress(["Scanning your index."])
         for index in self.index_config:
             index_name = index["index_name"]
             print(f"backup: export in progress for {index_name}")
@@ -41,6 +42,7 @@ class ElasticBackup:
 
             self.backup_index(index_name)
 
+        self.task.send_progress(["Compress files to zip archive."])
         self.zip_it()
         if self.reason == "auto":
             self.rotate_backup()

tubearchivist/home/src/ta/helper.py

@@ -11,6 +11,7 @@ import string
 import subprocess
 import unicodedata
 from datetime import datetime
+from urllib.parse import urlparse
 
 import requests
 
@@ -148,6 +149,22 @@ def is_shorts(youtube_id):
     return response.status_code == 200
 
 
+def ta_host_parser(ta_host):
+    """parse ta_host env var for ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS"""
+    allowed_hosts = []
+    csrf_trusted_origins = []
+    for host in ta_host.split():
+        host_clean = host.strip()
+        if not host_clean.startswith("http"):
+            host_clean = f"http://{host_clean}"
+
+        parsed = urlparse(host_clean)
+        allowed_hosts.append(f"{parsed.hostname}")
+        csrf_trusted_origins.append(f"{parsed.scheme}://{parsed.hostname}")
+
+    return allowed_hosts, csrf_trusted_origins
+
+
 class DurationConverter:
     """
     using ffmpeg to get and parse duration from filepath

tubearchivist/requirements.txt

@@ -1,4 +1,4 @@
-beautifulsoup4==4.12.0
+beautifulsoup4==4.12.2
 celery==5.2.7
 Django==4.2
 django-auth-ldap==4.2.0