Merge branch 'testing' of https://github.com/bbilly1/tubearchivist into feat/react-frontend

tubearchivist/home/src/download/thumbnails.py

@@ -4,8 +4,10 @@ functionality:
 - check for missing thumbnails
 """
 
+import base64
 import os
 from collections import Counter
+from io import BytesIO
 from time import sleep
 
 import requests
@@ -15,7 +17,7 @@ from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
 from home.src.ta.ta_redis import RedisArchivist
 from mutagen.mp4 import MP4, MP4Cover
-from PIL import Image
+from PIL import Image, ImageFilter
 
 
 class ThumbManager:
@@ -241,6 +243,21 @@ class ThumbManager:
             }
             RedisArchivist().set_message("message:download", mess_dict)
 
+    def get_base64_blur(self, youtube_id):
+        """return base64 encoded placeholder"""
+        img_path = self.vid_thumb_path(youtube_id)
+        file_path = os.path.join(self.CACHE_DIR, img_path)
+        img_raw = Image.open(file_path)
+        img_raw.thumbnail((img_raw.width // 20, img_raw.height // 20))
+        img_blur = img_raw.filter(ImageFilter.BLUR)
+        buffer = BytesIO()
+        img_blur.save(buffer, format="JPEG")
+        img_data = buffer.getvalue()
+        img_base64 = base64.b64encode(img_data).decode()
+        data_url = f"data:image/jpg;base64,{img_base64}"
+
+        return data_url
+
     @staticmethod
     def vid_thumb_path(youtube_id):
         """build expected path for video thumbnail from youtube_id"""

tubearchivist/home/src/es/index_mapping.json

@@ -73,6 +73,10 @@
                     "type": "text",
                     "index": false
                 },
+                "vid_thumb_base64": {
+                    "type": "text",
+                    "index": false
+                },
                 "date_downloaded": {
                     "type": "date"
                 },

tubearchivist/home/src/index/video.py

@@ -10,6 +10,7 @@ from datetime import datetime
 
 import requests
 from django.conf import settings
+from home.src.download.thumbnails import ThumbManager
 from home.src.es.connect import ElasticWrap
 from home.src.index import channel as ta_channel
 from home.src.index.generic import YouTubeItem
@@ -389,12 +390,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         upload_date_time = datetime.strptime(upload_date, "%Y%m%d")
         published = upload_date_time.strftime("%Y-%m-%d")
         last_refresh = int(datetime.now().strftime("%s"))
+        base64_blur = ThumbManager().get_base64_blur(self.youtube_id)
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
             "description": self.youtube_meta["description"],
             "category": self.youtube_meta["categories"],
             "vid_thumb_url": self.youtube_meta["thumbnail"],
+            "vid_thumb_base64": base64_blur,
             "tags": self.youtube_meta["tags"],
             "published": published,
             "vid_last_refresh": last_refresh,

tubearchivist/home/src/ta/config.py

@@ -83,33 +83,32 @@ class AppConfig:
 
     def update_config(self, form_post):
         """update config values from settings form"""
-        config = self.config
         for key, value in form_post.items():
-            to_write = value[0]
+            if not value and not isinstance(value, int):
-            if len(to_write):
+                continue
-                if to_write == "0":
-                    to_write = False
-                elif to_write == "1":
-                    to_write = True
-                elif to_write.isdigit():
-                    to_write = int(to_write)
 
-                config_dict, config_value = key.split("_", maxsplit=1)
+            if value in ["0", 0]:
-                config[config_dict][config_value] = to_write
+                to_write = False
+            elif value == "1":
+                to_write = True
+            else:
+                to_write = value
 
-        RedisArchivist().set_message("config", config, expire=False)
+            config_dict, config_value = key.split("_", maxsplit=1)
+            self.config[config_dict][config_value] = to_write
+
+        RedisArchivist().set_message("config", self.config, expire=False)
 
     @staticmethod
     def set_user_config(form_post, user_id):
         """set values in redis for user settings"""
         for key, value in form_post.items():
-            to_write = value[0]
+            if not value:
-            if len(to_write):
+                continue
-                if to_write.isdigit():
+
-                    to_write = int(to_write)
+            message = {"status": value}
-                message = {"status": to_write}
+            redis_key = f"{user_id}:{key}"
-                redis_key = f"{user_id}:{key}"
+            RedisArchivist().set_message(redis_key, message, expire=False)
-                RedisArchivist().set_message(redis_key, message, expire=False)
 
     def get_colors(self):
         """overwrite config if user has set custom values"""
@@ -172,12 +171,11 @@ class ScheduleBuilder:
         print("processing form, restart container for changes to take effect")
         redis_config = self.config
         for key, value in form_post.items():
-            to_check = value[0]
+            if key in self.SCHEDULES and value:
-            if key in self.SCHEDULES and to_check:
                 try:
-                    to_write = self.value_builder(key, to_check)
+                    to_write = self.value_builder(key, value)
                 except ValueError:
-                    print(f"failed: {key} {to_check}")
+                    print(f"failed: {key} {value}")
                     mess_dict = {
                         "status": "message:setting",
                         "level": "error",
@@ -188,8 +186,8 @@ class ScheduleBuilder:
                     return
 
                 redis_config["scheduler"][key] = to_write
-            if key in self.CONFIG and to_check:
+            if key in self.CONFIG and value:
-                redis_config["scheduler"][key] = int(to_check)
+                redis_config["scheduler"][key] = int(value)
         RedisArchivist().set_message("config", redis_config, expire=False)
         mess_dict = {
             "status": "message:setting",
@@ -199,29 +197,33 @@ class ScheduleBuilder:
         }
         RedisArchivist().set_message("message:setting", mess_dict)
 
-    def value_builder(self, key, to_check):
+    def value_builder(self, key, value):
         """validate single cron form entry and return cron dict"""
-        print(f"change schedule for {key} to {to_check}")
+        print(f"change schedule for {key} to {value}")
-        if to_check == "0":
+        if value == "0":
             # deactivate this schedule
             return False
-        if re.search(r"[\d]{1,2}\/[\d]{1,2}", to_check):
+        if re.search(r"[\d]{1,2}\/[\d]{1,2}", value):
             # number/number cron format will fail in celery
             print("number/number schedule formatting not supported")
             raise ValueError
 
         keys = ["minute", "hour", "day_of_week"]
-        if to_check == "auto":
+        if value == "auto":
             # set to sensible default
             values = self.SCHEDULES[key].split()
         else:
-            values = to_check.split()
+            values = value.split()
 
         if len(keys) != len(values):
-            print(f"failed to parse {to_check} for {key}")
+            print(f"failed to parse {value} for {key}")
             raise ValueError("invalid input")
 
         to_write = dict(zip(keys, values))
+        all_hours = [int(i) for i in re.split(r"\D+", to_write["hour"])]
+        if max(all_hours) > 23:
+            print("hour can't be greater than 23")
+            raise ValueError("invalid input")
         try:
             int(to_write["minute"])
         except ValueError as error:

tubearchivist/home/views.py

@@ -8,7 +8,6 @@ import json
 import urllib.parse
 from time import sleep
 
-from django import forms
 from django.conf import settings
 from django.contrib.auth import login
 from django.contrib.auth.forms import AuthenticationForm
@@ -802,23 +801,25 @@ class SettingsView(View):
     @staticmethod
     def post(request):
         """handle form post to update settings"""
+        user_form = UserSettingsForm(request.POST)
+        if user_form.is_valid():
+            user_form_post = user_form.cleaned_data
+            if any(user_form_post.values()):
+                AppConfig().set_user_config(user_form_post, request.user.id)
 
-        form_response = forms.Form(request.POST)
+        app_form = ApplicationSettingsForm(request.POST)
-        if form_response.is_valid():
+        if app_form.is_valid():
-            form_post = dict(request.POST)
+            app_form_post = app_form.cleaned_data
-            print(form_post)
+            if app_form_post:
-            del form_post["csrfmiddlewaretoken"]
+                print(app_form_post)
-            config_handler = AppConfig()
+                AppConfig().update_config(app_form_post)
-            if "application-settings" in form_post:
+
-                del form_post["application-settings"]
+        scheduler_form = SchedulerSettingsForm(request.POST)
-                config_handler.update_config(form_post)
+        if scheduler_form.is_valid():
-            elif "user-settings" in form_post:
+            scheduler_form_post = scheduler_form.cleaned_data
-                del form_post["user-settings"]
+            if any(scheduler_form_post.values()):
-                config_handler.set_user_config(form_post, request.user.id)
+                print(scheduler_form_post)
-            elif "scheduler-settings" in form_post:
+                ScheduleBuilder().update_schedule_conf(scheduler_form_post)
-                del form_post["scheduler-settings"]
-                print(form_post)
-                ScheduleBuilder().update_schedule_conf(form_post)
 
         sleep(1)
         return redirect("settings", permanent=True)