diff --git a/tubearchivist/api/src/search_processor.py b/tubearchivist/api/src/search_processor.py index 232474d..9f77b05 100644 --- a/tubearchivist/api/src/search_processor.py +++ b/tubearchivist/api/src/search_processor.py @@ -7,15 +7,14 @@ Functionality: import urllib.parse from home.src.download.thumbnails import ThumbManager -from home.src.ta.config import AppConfig from home.src.ta.helper import date_praser, get_duration_str +from home.src.ta.settings import EnvironmentSettings class SearchProcess: """process search results""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR def __init__(self, response): self.response = response diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index dc67e01..54c88fb 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -20,6 +20,7 @@ from home.src.index.playlist import YoutubePlaylist from home.src.index.reindex import ReindexProgress from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.ta.config import AppConfig, ReleaseVersion +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.urlparser import Parser @@ -56,7 +57,13 @@ class ApiBaseView(APIView): def __init__(self): super().__init__() - self.response = {"data": False, "config": AppConfig().config} + self.response = { + "data": False, + "config": { + "enable_cast": EnvironmentSettings.ENABLE_CAST, + "downloads": AppConfig().config["downloads"], + }, + } self.data = {"query": {"match_all": {}}} self.status_code = False self.context = False diff --git a/tubearchivist/config/management/commands/ta_envcheck.py b/tubearchivist/config/management/commands/ta_envcheck.py index 446962d..76c5ba1 100644 --- a/tubearchivist/config/management/commands/ta_envcheck.py +++ b/tubearchivist/config/management/commands/ta_envcheck.py @@ -11,6 +11,7 @@ import re from django.core.management.base import BaseCommand, CommandError from home.models import Account +from home.src.ta.settings import EnvironmentSettings LOGO = """ @@ -96,18 +97,14 @@ class Command(BaseCommand): def _elastic_user_overwrite(self): """check for ELASTIC_USER overwrite""" - self.stdout.write("[2] set default ES user") - if not os.environ.get("ELASTIC_USER"): - os.environ.setdefault("ELASTIC_USER", "elastic") - - env = os.environ.get("ELASTIC_USER") - + self.stdout.write("[2] check ES user overwrite") + env = EnvironmentSettings.ES_USER self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}")) def _ta_port_overwrite(self): """set TA_PORT overwrite for nginx""" self.stdout.write("[3] check TA_PORT overwrite") - overwrite = os.environ.get("TA_PORT") + overwrite = EnvironmentSettings.TA_PORT if not overwrite: self.stdout.write(self.style.SUCCESS(" TA_PORT is not set")) return @@ -125,7 +122,7 @@ class Command(BaseCommand): def _ta_uwsgi_overwrite(self): """set TA_UWSGI_PORT overwrite""" self.stdout.write("[4] check TA_UWSGI_PORT overwrite") - overwrite = os.environ.get("TA_UWSGI_PORT") + overwrite = EnvironmentSettings.TA_UWSGI_PORT if not overwrite: message = " TA_UWSGI_PORT is not set" self.stdout.write(self.style.SUCCESS(message)) @@ -151,7 +148,7 @@ class Command(BaseCommand): def _enable_cast_overwrite(self): """cast workaround, remove auth for static files in nginx""" self.stdout.write("[5] check ENABLE_CAST overwrite") - overwrite = os.environ.get("ENABLE_CAST") + overwrite = EnvironmentSettings.ENABLE_CAST if not overwrite: self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set")) return @@ -174,8 +171,8 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS(message)) return - name = os.environ.get("TA_USERNAME") - password = os.environ.get("TA_PASSWORD") + name = EnvironmentSettings.TA_USERNAME + password = EnvironmentSettings.TA_PASSWORD Account.objects.create_superuser(name, password) message = f" ✓ new superuser with name {name} created" self.stdout.write(self.style.SUCCESS(message)) diff --git a/tubearchivist/config/management/commands/ta_migpath.py b/tubearchivist/config/management/commands/ta_migpath.py index 607c58e..05f0b76 100644 --- a/tubearchivist/config/management/commands/ta_migpath.py +++ b/tubearchivist/config/management/commands/ta_migpath.py @@ -6,8 +6,8 @@ import shutil from django.core.management.base import BaseCommand from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings TOPIC = """ @@ -58,8 +58,7 @@ class FolderMigration: """migrate video archive folder""" def __init__(self): - self.config = AppConfig().config - self.videos = self.config["application"]["videos"] + self.videos = EnvironmentSettings.MEDIA_DIR self.bulk_list = [] def get_to_migrate(self): @@ -84,8 +83,8 @@ class FolderMigration: def create_folders(self, to_migrate): """create required channel folders""" - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} for channel_id in all_channel_ids: diff --git a/tubearchivist/config/management/commands/ta_startup.py b/tubearchivist/config/management/commands/ta_startup.py index 602731f..347e438 100644 --- a/tubearchivist/config/management/commands/ta_startup.py +++ b/tubearchivist/config/management/commands/ta_startup.py @@ -14,6 +14,7 @@ from home.src.es.snapshot import ElasticSnapshot from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.helper import clear_dl_cache +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.users import UserConfig @@ -69,7 +70,7 @@ class Command(BaseCommand): "playlists", "videos", ] - cache_dir = AppConfig().config["application"]["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR for folder in folders: folder_path = os.path.join(cache_dir, folder) os.makedirs(folder_path, exist_ok=True) @@ -119,8 +120,7 @@ class Command(BaseCommand): def _clear_dl_cache(self): """clear leftover files from dl cache""" self.stdout.write("[5] clear leftover files from dl cache") - config = AppConfig().config - leftover_files = clear_dl_cache(config) + leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR) if leftover_files: self.stdout.write( self.style.SUCCESS(f" ✓ cleared {leftover_files} files") @@ -152,7 +152,7 @@ class Command(BaseCommand): def _mig_set_streams(self): """migration: update from 0.3.5 to 0.3.6, set streams and media_size""" self.stdout.write("[MIGRATION] index streams and media size") - videos = AppConfig().config["application"]["videos"] + videos = EnvironmentSettings.MEDIA_DIR data = { "query": { "bool": {"must_not": [{"exists": {"field": "streams"}}]} diff --git a/tubearchivist/config/settings.py b/tubearchivist/config/settings.py index 0175985..f410655 100644 --- a/tubearchivist/config/settings.py +++ b/tubearchivist/config/settings.py @@ -17,8 +17,8 @@ from pathlib import Path import ldap from corsheaders.defaults import default_headers from django_auth_ldap.config import LDAPSearch -from home.src.ta.config import AppConfig from home.src.ta.helper import ta_host_parser +from home.src.ta.settings import EnvironmentSettings # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ -PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) +PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode()) SECRET_KEY = PW_HASH.hexdigest() # SECURITY WARNING: don't run with debug turned on in production! @@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")): # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases -CACHE_DIR = AppConfig().config["application"]["cache_dir"] +CACHE_DIR = EnvironmentSettings.CACHE_DIR DB_PATH = path.join(CACHE_DIR, "db.sqlite3") DATABASES = { "default": { @@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")): # https://docs.djangoproject.com/en/3.2/topics/i18n/ LANGUAGE_CODE = "en-us" -TIME_ZONE = environ.get("TZ") or "UTC" +TIME_ZONE = EnvironmentSettings.TZ USE_I18N = True USE_L10N = True USE_TZ = True diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index 26d3bf9..1b655f4 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -25,10 +25,6 @@ "integrate_sponsorblock": false }, "application": { - "app_root": "/app", - "cache_dir": "/cache", - "videos": "/youtube", - "enable_cast": false, "enable_snapshot": true }, "scheduler": { diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index ca498c5..896f603 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -11,7 +11,7 @@ from time import sleep import requests from home.src.es.connect import ElasticWrap, IndexPaginate -from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from mutagen.mp4 import MP4, MP4Cover from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError @@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True class ThumbManagerBase: """base class for thumbnail management""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR VIDEO_DIR = os.path.join(CACHE_DIR, "videos") CHANNEL_DIR = os.path.join(CACHE_DIR, "channels") PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists") @@ -70,7 +69,7 @@ class ThumbManagerBase: img_raw = Image.open(self.fallback) return img_raw - app_root = self.CONFIG["application"]["app_root"] + app_root = EnvironmentSettings.APP_DIR default_map = { "video": os.path.join( app_root, "static/img/default-video-thumb.jpg" @@ -380,9 +379,8 @@ class ThumbFilesystem: class EmbedCallback: """callback class to embed thumbnails""" - CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] - MEDIA_DIR = CONFIG["application"]["videos"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR + MEDIA_DIR = EnvironmentSettings.MEDIA_DIR FORMAT = MP4Cover.FORMAT_JPEG def __init__(self, source, index_name, counter=0): diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index b39601c..b1c8fa7 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -10,6 +10,7 @@ from http import cookiejar from io import StringIO import yt_dlp +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist @@ -86,6 +87,7 @@ class CookieHandler: def __init__(self, config): self.cookie_io = False self.config = config + self.cache_dir = EnvironmentSettings.CACHE_DIR def get(self): """get cookie io stream""" @@ -95,8 +97,9 @@ class CookieHandler: def import_cookie(self): """import cookie from file""" - cache_path = self.config["application"]["cache_dir"] - import_path = os.path.join(cache_path, "import", "cookies.google.txt") + import_path = os.path.join( + self.cache_dir, "import", "cookies.google.txt" + ) try: with open(import_path, encoding="utf-8") as cookie_file: diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 9a865ab..efcdb9b 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class DownloadPostProcess: @@ -153,6 +154,8 @@ class VideoDownloader: self.youtube_id_list = youtube_id_list self.task = task self.config = AppConfig().config + self.cache_dir = EnvironmentSettings.CACHE_DIR + self.media_dir = EnvironmentSettings.MEDIA_DIR self._build_obs() self.channels = set() self.videos = set() @@ -262,10 +265,7 @@ class VideoDownloader: """initial obs""" self.obs = { "merge_output_format": "mp4", - "outtmpl": ( - self.config["application"]["cache_dir"] - + "/download/%(id)s.mp4" - ), + "outtmpl": (self.cache_dir + "/download/%(id)s.mp4"), "progress_hooks": [self._progress_hook], "noprogress": True, "continuedl": True, @@ -340,7 +340,7 @@ class VideoDownloader: if format_overwrite: obs["format"] = format_overwrite - dl_cache = self.config["application"]["cache_dir"] + "/download/" + dl_cache = self.cache_dir + "/download/" # check if already in cache to continue from there all_cached = ignore_filelist(os.listdir(dl_cache)) @@ -370,20 +370,20 @@ class VideoDownloader: def move_to_archive(self, vid_dict): """move downloaded video from cache to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID # make folder - folder = os.path.join(videos, vid_dict["channel"]["channel_id"]) + folder = os.path.join( + self.media_dir, vid_dict["channel"]["channel_id"] + ) if not os.path.exists(folder): os.makedirs(folder) if host_uid and host_gid: os.chown(folder, host_uid, host_gid) # move media file media_file = vid_dict["youtube_id"] + ".mp4" - cache_dir = self.config["application"]["cache_dir"] - old_path = os.path.join(cache_dir, "download", media_file) - new_path = os.path.join(videos, vid_dict["media_url"]) + old_path = os.path.join(self.cache_dir, "download", media_file) + new_path = os.path.join(self.media_dir, vid_dict["media_url"]) # move media file and fix permission shutil.move(old_path, new_path, copy_function=shutil.copyfile) if host_uid and host_gid: diff --git a/tubearchivist/home/src/es/backup.py b/tubearchivist/home/src/es/backup.py index 3dc1cf5..b23592a 100644 --- a/tubearchivist/home/src/es/backup.py +++ b/tubearchivist/home/src/es/backup.py @@ -13,6 +13,7 @@ from datetime import datetime from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.ta.config import AppConfig from home.src.ta.helper import get_mapping, ignore_filelist +from home.src.ta.settings import EnvironmentSettings class ElasticBackup: @@ -22,7 +23,7 @@ class ElasticBackup: def __init__(self, reason=False, task=False): self.config = AppConfig().config - self.cache_dir = self.config["application"]["cache_dir"] + self.cache_dir = EnvironmentSettings.CACHE_DIR self.timestamp = datetime.now().strftime("%Y%m%d") self.index_config = get_mapping() self.reason = reason @@ -217,6 +218,7 @@ class BackupCallback: self.index_name = index_name self.counter = counter self.timestamp = datetime.now().strftime("%Y%m%d") + self.cache_dir = EnvironmentSettings.CACHE_DIR def run(self): """run the junk task""" @@ -243,9 +245,8 @@ class BackupCallback: def _write_es_json(self, file_content): """write nd-json file for es _bulk API to disk""" - cache_dir = AppConfig().config["application"]["cache_dir"] index = self.index_name.lstrip("ta_") file_name = f"es_{index}-{self.timestamp}-{self.counter}.json" - file_path = os.path.join(cache_dir, "backup", file_name) + file_path = os.path.join(self.cache_dir, "backup", file_name) with open(file_path, "a+", encoding="utf-8") as f: f.write(file_content) diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index a7c3ff5..f343b8e 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -6,11 +6,11 @@ functionality: # pylint: disable=missing-timeout import json -import os from typing import Any import requests import urllib3 +from home.src.ta.settings import EnvironmentSettings class ElasticWrap: @@ -18,16 +18,14 @@ class ElasticWrap: returns response json and status code tuple """ - ES_URL: str = str(os.environ.get("ES_URL")) - ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) - ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic") - ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) - def __init__(self, path: str): - self.url: str = f"{self.ES_URL}/{path}" - self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS) + self.url: str = f"{EnvironmentSettings.ES_URL}/{path}" + self.auth: tuple[str, str] = ( + EnvironmentSettings.ES_USER, + EnvironmentSettings.ES_PASS, + ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def get( @@ -43,7 +41,7 @@ class ElasticWrap: "timeout": timeout, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False if data: @@ -78,7 +76,7 @@ class ElasticWrap: } ) - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.post(self.url, **kwargs) @@ -103,7 +101,7 @@ class ElasticWrap: "auth": self.auth, } - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.put(self.url, **kwargs) @@ -130,7 +128,7 @@ class ElasticWrap: if data: kwargs["json"] = data - if self.ES_DISABLE_VERIFY_SSL: + if EnvironmentSettings.ES_DISABLE_VERIFY_SSL: kwargs["verify"] = False response = requests.delete(self.url, **kwargs) diff --git a/tubearchivist/home/src/es/snapshot.py b/tubearchivist/home/src/es/snapshot.py index 15fc82c..d77f076 100644 --- a/tubearchivist/home/src/es/snapshot.py +++ b/tubearchivist/home/src/es/snapshot.py @@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo from home.src.es.connect import ElasticWrap from home.src.ta.helper import get_mapping +from home.src.ta.settings import EnvironmentSettings class ElasticSnapshot: @@ -256,7 +257,7 @@ class ElasticSnapshot: expected_format = "%Y-%m-%dT%H:%M:%S.%fZ" date = datetime.strptime(date_utc, expected_format) local_datetime = date.replace(tzinfo=ZoneInfo("localtime")) - converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ"))) + converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ)) converted_str = converted.strftime("%Y-%m-%d %H:%M") return converted_str diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 1907062..02e5726 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.generic import YouTubeItem from home.src.index.playlist import YoutubePlaylist +from home.src.ta.settings import EnvironmentSettings class YoutubeChannel(YouTubeItem): @@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem): def _info_json_fallback(self): """read channel info.json for additional metadata""" info_json = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{self.youtube_id}.info.json", ) @@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem): def get_folder_path(self): """get folder where media files get stored""" folder_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel_id"], ) return folder_path diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index ecd3509..ab208c2 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -8,14 +8,14 @@ import os from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo, index_new_video -from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings class Scanner: """scan index and filesystem""" - VIDEOS: str = AppConfig().config["application"]["videos"] + VIDEOS: str = EnvironmentSettings.MEDIA_DIR def __init__(self, task=False) -> None: self.task = task diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index a5f624d..e18623e 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -26,7 +26,6 @@ class YouTubeItem: self.youtube_id = youtube_id self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.config = AppConfig().config - self.app_conf = self.config["application"] self.youtube_meta = False self.json_data = False diff --git a/tubearchivist/home/src/index/manual.py b/tubearchivist/home/src/index/manual.py index 3a19588..a25b061 100644 --- a/tubearchivist/home/src/index/manual.py +++ b/tubearchivist/home/src/index/manual.py @@ -16,6 +16,7 @@ from home.src.index.comments import CommentList from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig from home.src.ta.helper import ignore_filelist +from home.src.ta.settings import EnvironmentSettings from PIL import Image from yt_dlp.utils import ISO639Utils @@ -28,7 +29,7 @@ class ImportFolderScanner: """ CONFIG = AppConfig().config - CACHE_DIR = CONFIG["application"]["cache_dir"] + CACHE_DIR = EnvironmentSettings.CACHE_DIR IMPORT_DIR = os.path.join(CACHE_DIR, "import") """All extensions should be in lowercase until better handling is in place. @@ -433,9 +434,9 @@ class ManualImport: def _move_to_archive(self, json_data): """move identified media file to archive""" - videos = self.config["application"]["videos"] - host_uid = self.config["application"]["HOST_UID"] - host_gid = self.config["application"]["HOST_GID"] + videos = EnvironmentSettings.MEDIA_DIR + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID channel, file = os.path.split(json_data["media_url"]) channel_folder = os.path.join(videos, channel) @@ -472,7 +473,7 @@ class ManualImport: os.remove(subtitle_file) channel_info = os.path.join( - self.config["application"]["cache_dir"], + EnvironmentSettings.CACHE_DIR, "import", f"{json_data['channel']['channel_id']}.info.json", ) diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py index b89c00d..c25115c 100644 --- a/tubearchivist/home/src/index/reindex.py +++ b/tubearchivist/home/src/index/reindex.py @@ -19,6 +19,7 @@ from home.src.index.comments import Comments from home.src.index.playlist import YoutubePlaylist from home.src.index.video import YoutubeVideo from home.src.ta.config import AppConfig +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisQueue @@ -293,7 +294,7 @@ class Reindex(ReindexBase): # get new media_url = os.path.join( - self.config["application"]["videos"], es_meta["media_url"] + EnvironmentSettings.MEDIA_DIR, es_meta["media_url"] ) video.build_json(media_path=media_url) if not video.youtube_meta: @@ -328,7 +329,7 @@ class Reindex(ReindexBase): def _rename_media_file(self, media_url_is, media_url_should): """handle title change""" print(f"[reindex] fix media_url {media_url_is} to {media_url_should}") - videos = self.config["application"]["videos"] + videos = EnvironmentSettings.MEDIA_DIR old_path = os.path.join(videos, media_url_is) new_path = os.path.join(videos, media_url_should) os.rename(old_path, new_path) diff --git a/tubearchivist/home/src/index/subtitle.py b/tubearchivist/home/src/index/subtitle.py index 18af56a..b57ad55 100644 --- a/tubearchivist/home/src/index/subtitle.py +++ b/tubearchivist/home/src/index/subtitle.py @@ -12,6 +12,7 @@ from datetime import datetime import requests from home.src.es.connect import ElasticWrap from home.src.ta.helper import requests_headers +from home.src.ta.settings import EnvironmentSettings class YoutubeSubtitle: @@ -113,7 +114,7 @@ class YoutubeSubtitle: def download_subtitles(self, relevant_subtitles): """download subtitle files to archive""" - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR indexed = [] for subtitle in relevant_subtitles: dest_path = os.path.join(videos_base, subtitle["media_url"]) @@ -149,8 +150,8 @@ class YoutubeSubtitle: with open(dest_path, "w", encoding="utf-8") as subfile: subfile.write(subtitle_str) - host_uid = self.video.config["application"]["HOST_UID"] - host_gid = self.video.config["application"]["HOST_GID"] + host_uid = EnvironmentSettings.HOST_UID + host_gid = EnvironmentSettings.HOST_GID if host_uid and host_gid: os.chown(dest_path, host_uid, host_gid) @@ -162,7 +163,7 @@ class YoutubeSubtitle: def delete(self, subtitles=False): """delete subtitles from index and filesystem""" youtube_id = self.video.youtube_id - videos_base = self.video.config["application"]["videos"] + videos_base = EnvironmentSettings.MEDIA_DIR # delete files if subtitles: files = [i["media_url"] for i in subtitles] diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py index 606e32f..b414350 100644 --- a/tubearchivist/home/src/index/video.py +++ b/tubearchivist/home/src/index/video.py @@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_streams import MediaStreamExtractor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor +from home.src.ta.settings import EnvironmentSettings from home.src.ta.users import UserConfig from ryd_client import ryd_client @@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): def build_dl_cache_path(self): """find video path in dl cache""" - cache_dir = self.app_conf["cache_dir"] + cache_dir = EnvironmentSettings.CACHE_DIR video_id = self.json_data["youtube_id"] cache_path = f"{cache_dir}/download/{video_id}.mp4" if os.path.exists(cache_path): return cache_path channel_path = os.path.join( - self.app_conf["videos"], + EnvironmentSettings.MEDIA_DIR, self.json_data["channel"]["channel_id"], f"{video_id}.mp4", ) @@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle): if not self.json_data: raise FileNotFoundError - video_base = self.app_conf["videos"] + video_base = EnvironmentSettings.MEDIA_DIR media_url = self.json_data.get("media_url") file_path = os.path.join(video_base, media_url) try: diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index a32d083..c6e9e68 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -5,7 +5,6 @@ Functionality: """ import json -import os import re from random import randint from time import sleep @@ -28,7 +27,6 @@ class AppConfig: if not config: config = self.get_config_file() - config["application"].update(self.get_config_env()) return config def get_config_file(self): @@ -36,25 +34,8 @@ class AppConfig: with open("home/config.json", "r", encoding="utf-8") as f: config_file = json.load(f) - config_file["application"].update(self.get_config_env()) - return config_file - @staticmethod - def get_config_env(): - """read environment application variables. - - Connection to ES is managed in ElasticWrap and the - connection to Redis is managed in RedisArchivist.""" - - application = { - "HOST_UID": int(os.environ.get("HOST_UID", False)), - "HOST_GID": int(os.environ.get("HOST_GID", False)), - "enable_cast": bool(os.environ.get("ENABLE_CAST")), - } - - return application - @staticmethod def get_config_redis(): """read config json set from redis to overwrite defaults""" diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index db6e4b6..4143ed6 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float: return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds) -def clear_dl_cache(config: dict) -> int: +def clear_dl_cache(cache_dir: str) -> int: """clear leftover files from dl cache""" print("clear download cache") - cache_dir = os.path.join(config["application"]["cache_dir"], "download") - leftover_files = ignore_filelist(os.listdir(cache_dir)) + download_cache_dir = os.path.join(cache_dir, "download") + leftover_files = ignore_filelist(os.listdir(download_cache_dir)) for cached in leftover_files: - to_delete = os.path.join(cache_dir, cached) + to_delete = os.path.join(download_cache_dir, cached) os.remove(to_delete) return len(leftover_files) diff --git a/tubearchivist/home/src/ta/settings.py b/tubearchivist/home/src/ta/settings.py new file mode 100644 index 0000000..0798107 --- /dev/null +++ b/tubearchivist/home/src/ta/settings.py @@ -0,0 +1,39 @@ +""" +Functionality: +- read and write application config backed by ES +- encapsulate persistence of application properties +""" +import os + + +class EnvironmentSettings: + """ + Handle settings for the application that are driven from the environment. + These will not change when the user is using the application. + These settings are only provided only on startup. + """ + + HOST_UID: int = int(os.environ.get("HOST_UID", False)) + HOST_GID: int = int(os.environ.get("HOST_GID", False)) + ENABLE_CAST: bool = bool(os.environ.get("ENABLE_CAST")) + TZ: str = str(os.environ.get("TZ", "UTC")) + TA_PORT: int = int(os.environ.get("TA_PORT", False)) + TA_UWSGI_PORT: int = int(os.environ.get("TA_UWSGI_PORT", False)) + TA_USERNAME: str = str(os.environ.get("TA_USERNAME")) + TA_PASSWORD: str = str(os.environ.get("TA_PASSWORD")) + + # Application Paths + MEDIA_DIR: str = str(os.environ.get("TA_MEDIA_DIR", "/youtube")) + APP_DIR: str = str(os.environ.get("TA_APP_DIR", "/app")) + CACHE_DIR: str = str(os.environ.get("TA_CACHE_DIR", "/cache")) + + # Redis + REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) + REDIS_PORT: int = int(os.environ.get("REDIS_PORT", 6379)) + REDIS_NAME_SPACE: str = str(os.environ.get("REDIS_NAME_SPACE", "ta:")) + + # ElasticSearch + ES_URL: str = str(os.environ.get("ES_URL")) + ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD")) + ES_USER: str = str(os.environ.get("ELASTIC_USER", "elastic")) + ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL")) diff --git a/tubearchivist/home/src/ta/ta_redis.py b/tubearchivist/home/src/ta/ta_redis.py index 77de528..de821fb 100644 --- a/tubearchivist/home/src/ta/ta_redis.py +++ b/tubearchivist/home/src/ta/ta_redis.py @@ -6,20 +6,21 @@ functionality: """ import json -import os import redis +from home.src.ta.settings import EnvironmentSettings class RedisBase: """connection base for redis""" - REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) - REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379) - NAME_SPACE: str = "ta:" + NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE def __init__(self): - self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) + self.conn = redis.Redis( + host=EnvironmentSettings.REDIS_HOST, + port=EnvironmentSettings.REDIS_PORT, + ) class RedisArchivist(RedisBase): diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py index c337381..57b342c 100644 --- a/tubearchivist/home/src/ta/users.py +++ b/tubearchivist/home/src/ta/users.py @@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False): class UserConfig: - """Handle settings for an individual user - - Create getters and setters for usage in the application. - Although tedious it helps prevents everything caring about how properties - are persisted. Plus it allows us to save anytime any value is set. - """ + """Handle settings for an individual user""" _DEFAULT_USER_SETTINGS = UserConfigType( colors="dark", diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index a2c4cdb..88b0969 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.notify import Notifications +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.task_manager import TaskManager from home.src.ta.urlparser import Parser CONFIG = AppConfig().config -REDIS_HOST = os.environ.get("REDIS_HOST") -REDIS_PORT = os.environ.get("REDIS_PORT") or 6379 +REDIS_HOST = EnvironmentSettings.REDIS_HOST +REDIS_PORT = EnvironmentSettings.REDIS_PORT os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") app = Celery( @@ -39,9 +40,11 @@ app = Celery( backend=f"redis://{REDIS_HOST}:{REDIS_PORT}", result_extended=True, ) -app.config_from_object("django.conf:settings", namespace="ta:") +app.config_from_object( + "django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE +) app.autodiscover_tasks() -app.conf.timezone = os.environ.get("TZ") or "UTC" +app.conf.timezone = EnvironmentSettings.TZ class BaseTask(Task): diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index c3051bb..dae3d17 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -42,6 +42,7 @@ from home.src.index.reindex import ReindexProgress from home.src.index.video_constants import VideoTypeEnum from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.helper import time_parser +from home.src.ta.settings import EnvironmentSettings from home.src.ta.ta_redis import RedisArchivist from home.src.ta.users import UserConfig from home.tasks import index_channel_playlists, subscribe_to @@ -56,7 +57,6 @@ class ArchivistViewConfig(View): self.view_origin = view_origin self.user_id = False self.user_conf: UserConfig = False - self.default_conf = False self.context = False def get_all_view_styles(self): @@ -73,11 +73,10 @@ class ArchivistViewConfig(View): """build default context for every view""" self.user_id = user_id self.user_conf = UserConfig(self.user_id) - self.default_conf = AppConfig().config self.context = { "colors": self.user_conf.get_value("colors"), - "cast": self.default_conf["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "sort_by": self.user_conf.get_value("sort_by"), "sort_order": self.user_conf.get_value("sort_order"), "view_style": self.user_conf.get_value( @@ -877,7 +876,7 @@ class VideoView(MinView): "video": video_data, "playlist_nav": playlist_nav, "title": video_data.get("title"), - "cast": config_handler.config["application"]["enable_cast"], + "cast": EnvironmentSettings.ENABLE_CAST, "config": config_handler.config, "position": time_parser(request.GET.get("t")), "reindex": reindex.get("state"), diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index e6d3b86..cb40b26 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -470,7 +470,7 @@ function createPlayer(button) { // If cast integration is enabled create cast button let castButton = ''; - if (videoData.config.application.enable_cast) { + if (videoData.config.enable_cast) { castButton = ``; }