tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move the startup application settings to a new class (#571) 

* Move the startup application settings to a new class

* Replace settings methods with static fields

* Move Redis and ES configuration to the settings class

* Fix environment python imports

* Update envcheck to use the new settings
This commit is contained in:
Clark 2023-10-28 03:27:03 +00:00 committed by GitHub
parent 5165c3e34a
commit 4d111aff82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 153 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tubearchivist/api/src/search_processor.py
View File

@ -7,15 +7,14 @@ Functionality:
import urllib.parse import urllib.parse
from home.src.download.thumbnails import ThumbManager from home.src.download.thumbnails import ThumbManager
from home.src.ta.config import AppConfig
from home.src.ta.helper import date_praser, get_duration_str from home.src.ta.helper import date_praser, get_duration_str
from home.src.ta.settings import EnvironmentSettings
class SearchProcess: class SearchProcess:
"""process search results""" """process search results"""
CONFIG = AppConfig().config CACHE_DIR = EnvironmentSettings.CACHE_DIR
CACHE_DIR = CONFIG["application"]["cache_dir"]
def __init__(self, response): def __init__(self, response):
self.response = response self.response = response

9
tubearchivist/api/views.py
View File

@ -20,6 +20,7 @@ from home.src.index.playlist import YoutubePlaylist
from home.src.index.reindex import ReindexProgress from home.src.index.reindex import ReindexProgress
from home.src.index.video import SponsorBlock, YoutubeVideo from home.src.index.video import SponsorBlock, YoutubeVideo
from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskCommand, TaskManager from home.src.ta.task_manager import TaskCommand, TaskManager
from home.src.ta.urlparser import Parser from home.src.ta.urlparser import Parser
@ -56,7 +57,13 @@ class ApiBaseView(APIView):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.response = {"data": False, "config": AppConfig().config} self.response = {
"data": False,
"config": {
"enable_cast": EnvironmentSettings.ENABLE_CAST,
"downloads": AppConfig().config["downloads"],
},
}
self.data = {"query": {"match_all": {}}} self.data = {"query": {"match_all": {}}}
self.status_code = False self.status_code = False
self.context = False self.context = False

19
tubearchivist/config/management/commands/ta_envcheck.py
View File

@ -11,6 +11,7 @@ import re
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from home.models import Account from home.models import Account
from home.src.ta.settings import EnvironmentSettings
LOGO = """ LOGO = """
@ -96,18 +97,14 @@ class Command(BaseCommand):
def _elastic_user_overwrite(self): def _elastic_user_overwrite(self):
"""check for ELASTIC_USER overwrite""" """check for ELASTIC_USER overwrite"""
self.stdout.write("[2] set default ES user") self.stdout.write("[2] check ES user overwrite")
if not os.environ.get("ELASTIC_USER"): env = EnvironmentSettings.ES_USER
os.environ.setdefault("ELASTIC_USER", "elastic")
env = os.environ.get("ELASTIC_USER")
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}")) self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
def _ta_port_overwrite(self): def _ta_port_overwrite(self):
"""set TA_PORT overwrite for nginx""" """set TA_PORT overwrite for nginx"""
self.stdout.write("[3] check TA_PORT overwrite") self.stdout.write("[3] check TA_PORT overwrite")
overwrite = os.environ.get("TA_PORT") overwrite = EnvironmentSettings.TA_PORT
if not overwrite: if not overwrite:
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set")) self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
return return
@ -125,7 +122,7 @@ class Command(BaseCommand):
def _ta_uwsgi_overwrite(self): def _ta_uwsgi_overwrite(self):
"""set TA_UWSGI_PORT overwrite""" """set TA_UWSGI_PORT overwrite"""
self.stdout.write("[4] check TA_UWSGI_PORT overwrite") self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
overwrite = os.environ.get("TA_UWSGI_PORT") overwrite = EnvironmentSettings.TA_UWSGI_PORT
if not overwrite: if not overwrite:
message = " TA_UWSGI_PORT is not set" message = " TA_UWSGI_PORT is not set"
self.stdout.write(self.style.SUCCESS(message)) self.stdout.write(self.style.SUCCESS(message))
@ -151,7 +148,7 @@ class Command(BaseCommand):
def _enable_cast_overwrite(self): def _enable_cast_overwrite(self):
"""cast workaround, remove auth for static files in nginx""" """cast workaround, remove auth for static files in nginx"""
self.stdout.write("[5] check ENABLE_CAST overwrite") self.stdout.write("[5] check ENABLE_CAST overwrite")
overwrite = os.environ.get("ENABLE_CAST") overwrite = EnvironmentSettings.ENABLE_CAST
if not overwrite: if not overwrite:
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set")) self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
return return
@ -174,8 +171,8 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS(message)) self.stdout.write(self.style.SUCCESS(message))
return return
name = os.environ.get("TA_USERNAME") name = EnvironmentSettings.TA_USERNAME
password = os.environ.get("TA_PASSWORD") password = EnvironmentSettings.TA_PASSWORD
Account.objects.create_superuser(name, password) Account.objects.create_superuser(name, password)
message = f" ✓ new superuser with name {name} created" message = f" ✓ new superuser with name {name} created"
self.stdout.write(self.style.SUCCESS(message)) self.stdout.write(self.style.SUCCESS(message))

9
tubearchivist/config/management/commands/ta_migpath.py
View File

@ -6,8 +6,8 @@ import shutil
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
TOPIC = """ TOPIC = """
@ -58,8 +58,7 @@ class FolderMigration:
"""migrate video archive folder""" """migrate video archive folder"""
def __init__(self): def __init__(self):
self.config = AppConfig().config self.videos = EnvironmentSettings.MEDIA_DIR
self.videos = self.config["application"]["videos"]
self.bulk_list = [] self.bulk_list = []
def get_to_migrate(self): def get_to_migrate(self):
@ -84,8 +83,8 @@ class FolderMigration:
def create_folders(self, to_migrate): def create_folders(self, to_migrate):
"""create required channel folders""" """create required channel folders"""
host_uid = self.config["application"]["HOST_UID"] host_uid = EnvironmentSettings.HOST_UID
host_gid = self.config["application"]["HOST_GID"] host_gid = EnvironmentSettings.HOST_GID
all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate} all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
for channel_id in all_channel_ids: for channel_id in all_channel_ids:

8
tubearchivist/config/management/commands/ta_startup.py
View File

@ -14,6 +14,7 @@ from home.src.es.snapshot import ElasticSnapshot
from home.src.index.video_streams import MediaStreamExtractor from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.config import AppConfig, ReleaseVersion from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.helper import clear_dl_cache from home.src.ta.helper import clear_dl_cache
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager from home.src.ta.task_manager import TaskManager
from home.src.ta.users import UserConfig from home.src.ta.users import UserConfig
@ -69,7 +70,7 @@ class Command(BaseCommand):
"playlists", "playlists",
"videos", "videos",
] ]
cache_dir = AppConfig().config["application"]["cache_dir"] cache_dir = EnvironmentSettings.CACHE_DIR
for folder in folders: for folder in folders:
folder_path = os.path.join(cache_dir, folder) folder_path = os.path.join(cache_dir, folder)
os.makedirs(folder_path, exist_ok=True) os.makedirs(folder_path, exist_ok=True)
@ -119,8 +120,7 @@ class Command(BaseCommand):
def _clear_dl_cache(self): def _clear_dl_cache(self):
"""clear leftover files from dl cache""" """clear leftover files from dl cache"""
self.stdout.write("[5] clear leftover files from dl cache") self.stdout.write("[5] clear leftover files from dl cache")
config = AppConfig().config leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
leftover_files = clear_dl_cache(config)
if leftover_files: if leftover_files:
self.stdout.write( self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {leftover_files} files") self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
@ -152,7 +152,7 @@ class Command(BaseCommand):
def _mig_set_streams(self): def _mig_set_streams(self):
"""migration: update from 0.3.5 to 0.3.6, set streams and media_size""" """migration: update from 0.3.5 to 0.3.6, set streams and media_size"""
self.stdout.write("[MIGRATION] index streams and media size") self.stdout.write("[MIGRATION] index streams and media size")
videos = AppConfig().config["application"]["videos"] videos = EnvironmentSettings.MEDIA_DIR
data = { data = {
"query": { "query": {
"bool": {"must_not": [{"exists": {"field": "streams"}}]} "bool": {"must_not": [{"exists": {"field": "streams"}}]}

8
tubearchivist/config/settings.py
View File

@ -17,8 +17,8 @@ from pathlib import Path
import ldap import ldap
from corsheaders.defaults import default_headers from corsheaders.defaults import default_headers
from django_auth_ldap.config import LDAPSearch from django_auth_ldap.config import LDAPSearch
from home.src.ta.config import AppConfig
from home.src.ta.helper import ta_host_parser from home.src.ta.helper import ta_host_parser
from home.src.ta.settings import EnvironmentSettings
# Build paths inside the project like this: BASE_DIR / 'subdir'. # Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent BASE_DIR = Path(__file__).resolve().parent.parent
@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production # Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode()) PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
SECRET_KEY = PW_HASH.hexdigest() SECRET_KEY = PW_HASH.hexdigest()
# SECURITY WARNING: don't run with debug turned on in production! # SECURITY WARNING: don't run with debug turned on in production!
@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")):
# Database # Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
CACHE_DIR = AppConfig().config["application"]["cache_dir"] CACHE_DIR = EnvironmentSettings.CACHE_DIR
DB_PATH = path.join(CACHE_DIR, "db.sqlite3") DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
DATABASES = { DATABASES = {
"default": { "default": {
@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
# https://docs.djangoproject.com/en/3.2/topics/i18n/ # https://docs.djangoproject.com/en/3.2/topics/i18n/
LANGUAGE_CODE = "en-us" LANGUAGE_CODE = "en-us"
TIME_ZONE = environ.get("TZ") or "UTC" TIME_ZONE = EnvironmentSettings.TZ
USE_I18N = True USE_I18N = True
USE_L10N = True USE_L10N = True
USE_TZ = True USE_TZ = True

4
tubearchivist/home/config.json
View File

@ -25,10 +25,6 @@
"integrate_sponsorblock": false "integrate_sponsorblock": false
}, },
"application": { "application": {
"app_root": "/app",
"cache_dir": "/cache",
"videos": "/youtube",
"enable_cast": false,
"enable_snapshot": true "enable_snapshot": true
}, },
"scheduler": { "scheduler": {

12
tubearchivist/home/src/download/thumbnails.py
View File

@ -11,7 +11,7 @@ from time import sleep
import requests import requests
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig from home.src.ta.settings import EnvironmentSettings
from mutagen.mp4 import MP4, MP4Cover from mutagen.mp4 import MP4, MP4Cover
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
class ThumbManagerBase: class ThumbManagerBase:
"""base class for thumbnail management""" """base class for thumbnail management"""
CONFIG = AppConfig().config CACHE_DIR = EnvironmentSettings.CACHE_DIR
CACHE_DIR = CONFIG["application"]["cache_dir"]
VIDEO_DIR = os.path.join(CACHE_DIR, "videos") VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels") CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists") PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
@ -70,7 +69,7 @@ class ThumbManagerBase:
img_raw = Image.open(self.fallback) img_raw = Image.open(self.fallback)
return img_raw return img_raw
app_root = self.CONFIG["application"]["app_root"] app_root = EnvironmentSettings.APP_DIR
default_map = { default_map = {
"video": os.path.join( "video": os.path.join(
app_root, "static/img/default-video-thumb.jpg" app_root, "static/img/default-video-thumb.jpg"
@ -380,9 +379,8 @@ class ThumbFilesystem:
class EmbedCallback: class EmbedCallback:
"""callback class to embed thumbnails""" """callback class to embed thumbnails"""
CONFIG = AppConfig().config CACHE_DIR = EnvironmentSettings.CACHE_DIR
CACHE_DIR = CONFIG["application"]["cache_dir"] MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
MEDIA_DIR = CONFIG["application"]["videos"]
FORMAT = MP4Cover.FORMAT_JPEG FORMAT = MP4Cover.FORMAT_JPEG
def __init__(self, source, index_name, counter=0): def __init__(self, source, index_name, counter=0):

7
tubearchivist/home/src/download/yt_dlp_base.py
View File

@ -10,6 +10,7 @@ from http import cookiejar
from io import StringIO from io import StringIO
import yt_dlp import yt_dlp
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
@ -86,6 +87,7 @@ class CookieHandler:
def __init__(self, config): def __init__(self, config):
self.cookie_io = False self.cookie_io = False
self.config = config self.config = config
self.cache_dir = EnvironmentSettings.CACHE_DIR
def get(self): def get(self):
"""get cookie io stream""" """get cookie io stream"""
@ -95,8 +97,9 @@ class CookieHandler:
def import_cookie(self): def import_cookie(self):
"""import cookie from file""" """import cookie from file"""
cache_path = self.config["application"]["cache_dir"] import_path = os.path.join(
import_path = os.path.join(cache_path, "import", "cookies.google.txt") self.cache_dir, "import", "cookies.google.txt"
)
try: try:
with open(import_path, encoding="utf-8") as cookie_file: with open(import_path, encoding="utf-8") as cookie_file:

24
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video
from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class DownloadPostProcess: class DownloadPostProcess:
@ -153,6 +154,8 @@ class VideoDownloader:
self.youtube_id_list = youtube_id_list self.youtube_id_list = youtube_id_list
self.task = task self.task = task
self.config = AppConfig().config self.config = AppConfig().config
self.cache_dir = EnvironmentSettings.CACHE_DIR
self.media_dir = EnvironmentSettings.MEDIA_DIR
self._build_obs() self._build_obs()
self.channels = set() self.channels = set()
self.videos = set() self.videos = set()
@ -262,10 +265,7 @@ class VideoDownloader:
"""initial obs""" """initial obs"""
self.obs = { self.obs = {
"merge_output_format": "mp4", "merge_output_format": "mp4",
"outtmpl": ( "outtmpl": (self.cache_dir + "/download/%(id)s.mp4"),
self.config["application"]["cache_dir"]
+ "/download/%(id)s.mp4"
),
"progress_hooks": [self._progress_hook], "progress_hooks": [self._progress_hook],
"noprogress": True, "noprogress": True,
"continuedl": True, "continuedl": True,
@ -340,7 +340,7 @@ class VideoDownloader:
if format_overwrite: if format_overwrite:
obs["format"] = format_overwrite obs["format"] = format_overwrite
dl_cache = self.config["application"]["cache_dir"] + "/download/" dl_cache = self.cache_dir + "/download/"
# check if already in cache to continue from there # check if already in cache to continue from there
all_cached = ignore_filelist(os.listdir(dl_cache)) all_cached = ignore_filelist(os.listdir(dl_cache))
@ -370,20 +370,20 @@ class VideoDownloader:
def move_to_archive(self, vid_dict): def move_to_archive(self, vid_dict):
"""move downloaded video from cache to archive""" """move downloaded video from cache to archive"""
videos = self.config["application"]["videos"] host_uid = EnvironmentSettings.HOST_UID
host_uid = self.config["application"]["HOST_UID"] host_gid = EnvironmentSettings.HOST_GID
host_gid = self.config["application"]["HOST_GID"]
# make folder # make folder
folder = os.path.join(videos, vid_dict["channel"]["channel_id"]) folder = os.path.join(
self.media_dir, vid_dict["channel"]["channel_id"]
)
if not os.path.exists(folder): if not os.path.exists(folder):
os.makedirs(folder) os.makedirs(folder)
if host_uid and host_gid: if host_uid and host_gid:
os.chown(folder, host_uid, host_gid) os.chown(folder, host_uid, host_gid)
# move media file # move media file
media_file = vid_dict["youtube_id"] + ".mp4" media_file = vid_dict["youtube_id"] + ".mp4"
cache_dir = self.config["application"]["cache_dir"] old_path = os.path.join(self.cache_dir, "download", media_file)
old_path = os.path.join(cache_dir, "download", media_file) new_path = os.path.join(self.media_dir, vid_dict["media_url"])
new_path = os.path.join(videos, vid_dict["media_url"])
# move media file and fix permission # move media file and fix permission
shutil.move(old_path, new_path, copy_function=shutil.copyfile) shutil.move(old_path, new_path, copy_function=shutil.copyfile)
if host_uid and host_gid: if host_uid and host_gid:

7
tubearchivist/home/src/es/backup.py
View File

@ -13,6 +13,7 @@ from datetime import datetime
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.helper import get_mapping, ignore_filelist from home.src.ta.helper import get_mapping, ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class ElasticBackup: class ElasticBackup:
@ -22,7 +23,7 @@ class ElasticBackup:
def __init__(self, reason=False, task=False): def __init__(self, reason=False, task=False):
self.config = AppConfig().config self.config = AppConfig().config
self.cache_dir = self.config["application"]["cache_dir"] self.cache_dir = EnvironmentSettings.CACHE_DIR
self.timestamp = datetime.now().strftime("%Y%m%d") self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = get_mapping() self.index_config = get_mapping()
self.reason = reason self.reason = reason
@ -217,6 +218,7 @@ class BackupCallback:
self.index_name = index_name self.index_name = index_name
self.counter = counter self.counter = counter
self.timestamp = datetime.now().strftime("%Y%m%d") self.timestamp = datetime.now().strftime("%Y%m%d")
self.cache_dir = EnvironmentSettings.CACHE_DIR
def run(self): def run(self):
"""run the junk task""" """run the junk task"""
@ -243,9 +245,8 @@ class BackupCallback:
def _write_es_json(self, file_content): def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk""" """write nd-json file for es _bulk API to disk"""
cache_dir = AppConfig().config["application"]["cache_dir"]
index = self.index_name.lstrip("ta_") index = self.index_name.lstrip("ta_")
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json" file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
file_path = os.path.join(cache_dir, "backup", file_name) file_path = os.path.join(self.cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f: with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content) f.write(file_content)

24
tubearchivist/home/src/es/connect.py
View File

@ -6,11 +6,11 @@ functionality:
# pylint: disable=missing-timeout # pylint: disable=missing-timeout
import json import json
import os
from typing import Any from typing import Any
import requests import requests
import urllib3 import urllib3
from home.src.ta.settings import EnvironmentSettings
class ElasticWrap: class ElasticWrap:
@ -18,16 +18,14 @@ class ElasticWrap:
returns response json and status code tuple returns response json and status code tuple
""" """
ES_URL: str = str(os.environ.get("ES_URL"))
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))
def __init__(self, path: str): def __init__(self, path: str):
self.url: str = f"{self.ES_URL}/{path}" self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS) self.auth: tuple[str, str] = (
EnvironmentSettings.ES_USER,
EnvironmentSettings.ES_PASS,
)
if self.ES_DISABLE_VERIFY_SSL: if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def get( def get(
@ -43,7 +41,7 @@ class ElasticWrap:
"timeout": timeout, "timeout": timeout,
} }
if self.ES_DISABLE_VERIFY_SSL: if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False kwargs["verify"] = False
if data: if data:
@ -78,7 +76,7 @@ class ElasticWrap:
} }
) )
if self.ES_DISABLE_VERIFY_SSL: if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False kwargs["verify"] = False
response = requests.post(self.url, **kwargs) response = requests.post(self.url, **kwargs)
@ -103,7 +101,7 @@ class ElasticWrap:
"auth": self.auth, "auth": self.auth,
} }
if self.ES_DISABLE_VERIFY_SSL: if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False kwargs["verify"] = False
response = requests.put(self.url, **kwargs) response = requests.put(self.url, **kwargs)
@ -130,7 +128,7 @@ class ElasticWrap:
if data: if data:
kwargs["json"] = data kwargs["json"] = data
if self.ES_DISABLE_VERIFY_SSL: if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False kwargs["verify"] = False
response = requests.delete(self.url, **kwargs) response = requests.delete(self.url, **kwargs)

3
tubearchivist/home/src/es/snapshot.py
View File

@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo
from home.src.es.connect import ElasticWrap from home.src.es.connect import ElasticWrap
from home.src.ta.helper import get_mapping from home.src.ta.helper import get_mapping
from home.src.ta.settings import EnvironmentSettings
class ElasticSnapshot: class ElasticSnapshot:
@ -256,7 +257,7 @@ class ElasticSnapshot:
expected_format = "%Y-%m-%dT%H:%M:%S.%fZ" expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
date = datetime.strptime(date_utc, expected_format) date = datetime.strptime(date_utc, expected_format)
local_datetime = date.replace(tzinfo=ZoneInfo("localtime")) local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ"))) converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
converted_str = converted.strftime("%Y-%m-%d %H:%M") converted_str = converted.strftime("%Y-%m-%d %H:%M")
return converted_str return converted_str

5
tubearchivist/home/src/index/channel.py
View File

@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.generic import YouTubeItem from home.src.index.generic import YouTubeItem
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.ta.settings import EnvironmentSettings
class YoutubeChannel(YouTubeItem): class YoutubeChannel(YouTubeItem):
@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem):
def _info_json_fallback(self): def _info_json_fallback(self):
"""read channel info.json for additional metadata""" """read channel info.json for additional metadata"""
info_json = os.path.join( info_json = os.path.join(
self.config["application"]["cache_dir"], EnvironmentSettings.CACHE_DIR,
"import", "import",
f"{self.youtube_id}.info.json", f"{self.youtube_id}.info.json",
) )
@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem):
def get_folder_path(self): def get_folder_path(self):
"""get folder where media files get stored""" """get folder where media files get stored"""
folder_path = os.path.join( folder_path = os.path.join(
self.app_conf["videos"], EnvironmentSettings.MEDIA_DIR,
self.json_data["channel_id"], self.json_data["channel_id"],
) )
return folder_path return folder_path

4
tubearchivist/home/src/index/filesystem.py
View File

@ -8,14 +8,14 @@ import os
from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.comments import CommentList from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo, index_new_video from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class Scanner: class Scanner:
"""scan index and filesystem""" """scan index and filesystem"""
VIDEOS: str = AppConfig().config["application"]["videos"] VIDEOS: str = EnvironmentSettings.MEDIA_DIR
def __init__(self, task=False) -> None: def __init__(self, task=False) -> None:
self.task = task self.task = task

1
tubearchivist/home/src/index/generic.py
View File

@ -26,7 +26,6 @@ class YouTubeItem:
self.youtube_id = youtube_id self.youtube_id = youtube_id
self.es_path = f"{self.index_name}/_doc/{youtube_id}" self.es_path = f"{self.index_name}/_doc/{youtube_id}"
self.config = AppConfig().config self.config = AppConfig().config
self.app_conf = self.config["application"]
self.youtube_meta = False self.youtube_meta = False
self.json_data = False self.json_data = False

11
tubearchivist/home/src/index/manual.py
View File

@ -16,6 +16,7 @@ from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
from PIL import Image from PIL import Image
from yt_dlp.utils import ISO639Utils from yt_dlp.utils import ISO639Utils
@ -28,7 +29,7 @@ class ImportFolderScanner:
""" """
CONFIG = AppConfig().config CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"] CACHE_DIR = EnvironmentSettings.CACHE_DIR
IMPORT_DIR = os.path.join(CACHE_DIR, "import") IMPORT_DIR = os.path.join(CACHE_DIR, "import")
"""All extensions should be in lowercase until better handling is in place. """All extensions should be in lowercase until better handling is in place.
@ -433,9 +434,9 @@ class ManualImport:
def _move_to_archive(self, json_data): def _move_to_archive(self, json_data):
"""move identified media file to archive""" """move identified media file to archive"""
videos = self.config["application"]["videos"] videos = EnvironmentSettings.MEDIA_DIR
host_uid = self.config["application"]["HOST_UID"] host_uid = EnvironmentSettings.HOST_UID
host_gid = self.config["application"]["HOST_GID"] host_gid = EnvironmentSettings.HOST_GID
channel, file = os.path.split(json_data["media_url"]) channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel) channel_folder = os.path.join(videos, channel)
@ -472,7 +473,7 @@ class ManualImport:
os.remove(subtitle_file) os.remove(subtitle_file)
channel_info = os.path.join( channel_info = os.path.join(
self.config["application"]["cache_dir"], EnvironmentSettings.CACHE_DIR,
"import", "import",
f"{json_data['channel']['channel_id']}.info.json", f"{json_data['channel']['channel_id']}.info.json",
) )

5
tubearchivist/home/src/index/reindex.py
View File

@ -19,6 +19,7 @@ from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig from home.src.ta.config import AppConfig
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisQueue from home.src.ta.ta_redis import RedisQueue
@ -293,7 +294,7 @@ class Reindex(ReindexBase):
# get new # get new
media_url = os.path.join( media_url = os.path.join(
self.config["application"]["videos"], es_meta["media_url"] EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
) )
video.build_json(media_path=media_url) video.build_json(media_path=media_url)
if not video.youtube_meta: if not video.youtube_meta:
@ -328,7 +329,7 @@ class Reindex(ReindexBase):
def _rename_media_file(self, media_url_is, media_url_should): def _rename_media_file(self, media_url_is, media_url_should):
"""handle title change""" """handle title change"""
print(f"[reindex] fix media_url {media_url_is} to {media_url_should}") print(f"[reindex] fix media_url {media_url_is} to {media_url_should}")
videos = self.config["application"]["videos"] videos = EnvironmentSettings.MEDIA_DIR
old_path = os.path.join(videos, media_url_is) old_path = os.path.join(videos, media_url_is)
new_path = os.path.join(videos, media_url_should) new_path = os.path.join(videos, media_url_should)
os.rename(old_path, new_path) os.rename(old_path, new_path)

9
tubearchivist/home/src/index/subtitle.py
View File

@ -12,6 +12,7 @@ from datetime import datetime
import requests import requests
from home.src.es.connect import ElasticWrap from home.src.es.connect import ElasticWrap
from home.src.ta.helper import requests_headers from home.src.ta.helper import requests_headers
from home.src.ta.settings import EnvironmentSettings
class YoutubeSubtitle: class YoutubeSubtitle:
@ -113,7 +114,7 @@ class YoutubeSubtitle:
def download_subtitles(self, relevant_subtitles): def download_subtitles(self, relevant_subtitles):
"""download subtitle files to archive""" """download subtitle files to archive"""
videos_base = self.video.config["application"]["videos"] videos_base = EnvironmentSettings.MEDIA_DIR
indexed = [] indexed = []
for subtitle in relevant_subtitles: for subtitle in relevant_subtitles:
dest_path = os.path.join(videos_base, subtitle["media_url"]) dest_path = os.path.join(videos_base, subtitle["media_url"])
@ -149,8 +150,8 @@ class YoutubeSubtitle:
with open(dest_path, "w", encoding="utf-8") as subfile: with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(subtitle_str) subfile.write(subtitle_str)
host_uid = self.video.config["application"]["HOST_UID"] host_uid = EnvironmentSettings.HOST_UID
host_gid = self.video.config["application"]["HOST_GID"] host_gid = EnvironmentSettings.HOST_GID
if host_uid and host_gid: if host_uid and host_gid:
os.chown(dest_path, host_uid, host_gid) os.chown(dest_path, host_uid, host_gid)
@ -162,7 +163,7 @@ class YoutubeSubtitle:
def delete(self, subtitles=False): def delete(self, subtitles=False):
"""delete subtitles from index and filesystem""" """delete subtitles from index and filesystem"""
youtube_id = self.video.youtube_id youtube_id = self.video.youtube_id
videos_base = self.video.config["application"]["videos"] videos_base = EnvironmentSettings.MEDIA_DIR
# delete files # delete files
if subtitles: if subtitles:
files = [i["media_url"] for i in subtitles] files = [i["media_url"] for i in subtitles]

7
tubearchivist/home/src/index/video.py
View File

@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle
from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_constants import VideoTypeEnum
from home.src.index.video_streams import MediaStreamExtractor from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.users import UserConfig from home.src.ta.users import UserConfig
from ryd_client import ryd_client from ryd_client import ryd_client
@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
def build_dl_cache_path(self): def build_dl_cache_path(self):
"""find video path in dl cache""" """find video path in dl cache"""
cache_dir = self.app_conf["cache_dir"] cache_dir = EnvironmentSettings.CACHE_DIR
video_id = self.json_data["youtube_id"] video_id = self.json_data["youtube_id"]
cache_path = f"{cache_dir}/download/{video_id}.mp4" cache_path = f"{cache_dir}/download/{video_id}.mp4"
if os.path.exists(cache_path): if os.path.exists(cache_path):
return cache_path return cache_path
channel_path = os.path.join( channel_path = os.path.join(
self.app_conf["videos"], EnvironmentSettings.MEDIA_DIR,
self.json_data["channel"]["channel_id"], self.json_data["channel"]["channel_id"],
f"{video_id}.mp4", f"{video_id}.mp4",
) )
@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
if not self.json_data: if not self.json_data:
raise FileNotFoundError raise FileNotFoundError
video_base = self.app_conf["videos"] video_base = EnvironmentSettings.MEDIA_DIR
media_url = self.json_data.get("media_url") media_url = self.json_data.get("media_url")
file_path = os.path.join(video_base, media_url) file_path = os.path.join(video_base, media_url)
try: try:

19
tubearchivist/home/src/ta/config.py
View File

@ -5,7 +5,6 @@ Functionality:
""" """
import json import json
import os
import re import re
from random import randint from random import randint
from time import sleep from time import sleep
@ -28,7 +27,6 @@ class AppConfig:
if not config: if not config:
config = self.get_config_file() config = self.get_config_file()
config["application"].update(self.get_config_env())
return config return config
def get_config_file(self): def get_config_file(self):
@ -36,25 +34,8 @@ class AppConfig:
with open("home/config.json", "r", encoding="utf-8") as f: with open("home/config.json", "r", encoding="utf-8") as f:
config_file = json.load(f) config_file = json.load(f)
config_file["application"].update(self.get_config_env())
return config_file return config_file
@staticmethod
def get_config_env():
"""read environment application variables.
Connection to ES is managed in ElasticWrap and the
connection to Redis is managed in RedisArchivist."""
application = {
"HOST_UID": int(os.environ.get("HOST_UID", False)),
"HOST_GID": int(os.environ.get("HOST_GID", False)),
"enable_cast": bool(os.environ.get("ENABLE_CAST")),
}
return application
@staticmethod @staticmethod
def get_config_redis(): def get_config_redis():
"""read config json set from redis to overwrite defaults""" """read config json set from redis to overwrite defaults"""

8
tubearchivist/home/src/ta/helper.py
View File

@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float:
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds) return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
def clear_dl_cache(config: dict) -> int: def clear_dl_cache(cache_dir: str) -> int:
"""clear leftover files from dl cache""" """clear leftover files from dl cache"""
print("clear download cache") print("clear download cache")
cache_dir = os.path.join(config["application"]["cache_dir"], "download") download_cache_dir = os.path.join(cache_dir, "download")
leftover_files = ignore_filelist(os.listdir(cache_dir)) leftover_files = ignore_filelist(os.listdir(download_cache_dir))
for cached in leftover_files: for cached in leftover_files:
to_delete = os.path.join(cache_dir, cached) to_delete = os.path.join(download_cache_dir, cached)
os.remove(to_delete) os.remove(to_delete)
return len(leftover_files) return len(leftover_files)

39
tubearchivist/home/src/ta/settings.py Normal file
View File

@ -0,0 +1,39 @@
"""
Functionality:
- read and write application config backed by ES
- encapsulate persistence of application properties
"""
import os
class EnvironmentSettings:
"""
Handle settings for the application that are driven from the environment.
These will not change when the user is using the application.
These settings are only provided only on startup.
"""
HOST_UID: int = int(os.environ.get("HOST_UID", False))
HOST_GID: int = int(os.environ.get("HOST_GID", False))
ENABLE_CAST: bool = bool(os.environ.get("ENABLE_CAST"))
TZ: str = str(os.environ.get("TZ", "UTC"))
TA_PORT: int = int(os.environ.get("TA_PORT", False))
TA_UWSGI_PORT: int = int(os.environ.get("TA_UWSGI_PORT", False))
TA_USERNAME: str = str(os.environ.get("TA_USERNAME"))
TA_PASSWORD: str = str(os.environ.get("TA_PASSWORD"))
# Application Paths
MEDIA_DIR: str = str(os.environ.get("TA_MEDIA_DIR", "/youtube"))
APP_DIR: str = str(os.environ.get("TA_APP_DIR", "/app"))
CACHE_DIR: str = str(os.environ.get("TA_CACHE_DIR", "/cache"))
# Redis
REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
REDIS_PORT: int = int(os.environ.get("REDIS_PORT", 6379))
REDIS_NAME_SPACE: str = str(os.environ.get("REDIS_NAME_SPACE", "ta:"))
# ElasticSearch
ES_URL: str = str(os.environ.get("ES_URL"))
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(os.environ.get("ELASTIC_USER", "elastic"))
ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))

11
tubearchivist/home/src/ta/ta_redis.py
View File

@ -6,20 +6,21 @@ functionality:
""" """
import json import json
import os
import redis import redis
from home.src.ta.settings import EnvironmentSettings
class RedisBase: class RedisBase:
"""connection base for redis""" """connection base for redis"""
REDIS_HOST: str = str(os.environ.get("REDIS_HOST")) NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
NAME_SPACE: str = "ta:"
def __init__(self): def __init__(self):
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) self.conn = redis.Redis(
host=EnvironmentSettings.REDIS_HOST,
port=EnvironmentSettings.REDIS_PORT,
)
class RedisArchivist(RedisBase): class RedisArchivist(RedisBase):

7
tubearchivist/home/src/ta/users.py
View File

@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False):
class UserConfig: class UserConfig:
"""Handle settings for an individual user """Handle settings for an individual user"""
Create getters and setters for usage in the application.
Although tedious it helps prevents everything caring about how properties
are persisted. Plus it allows us to save anytime any value is set.
"""
_DEFAULT_USER_SETTINGS = UserConfigType( _DEFAULT_USER_SETTINGS = UserConfigType(
colors="dark", colors="dark",

11
tubearchivist/home/tasks.py
View File

@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.notify import Notifications from home.src.ta.notify import Notifications
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager from home.src.ta.task_manager import TaskManager
from home.src.ta.urlparser import Parser from home.src.ta.urlparser import Parser
CONFIG = AppConfig().config CONFIG = AppConfig().config
REDIS_HOST = os.environ.get("REDIS_HOST") REDIS_HOST = EnvironmentSettings.REDIS_HOST
REDIS_PORT = os.environ.get("REDIS_PORT") or 6379 REDIS_PORT = EnvironmentSettings.REDIS_PORT
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
app = Celery( app = Celery(
@ -39,9 +40,11 @@ app = Celery(
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}", backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
result_extended=True, result_extended=True,
) )
app.config_from_object("django.conf:settings", namespace="ta:") app.config_from_object(
"django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
)
app.autodiscover_tasks() app.autodiscover_tasks()
app.conf.timezone = os.environ.get("TZ") or "UTC" app.conf.timezone = EnvironmentSettings.TZ
class BaseTask(Task): class BaseTask(Task):

7
tubearchivist/home/views.py
View File

@ -42,6 +42,7 @@ from home.src.index.reindex import ReindexProgress
from home.src.index.video_constants import VideoTypeEnum from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.helper import time_parser from home.src.ta.helper import time_parser
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.users import UserConfig from home.src.ta.users import UserConfig
from home.tasks import index_channel_playlists, subscribe_to from home.tasks import index_channel_playlists, subscribe_to
@ -56,7 +57,6 @@ class ArchivistViewConfig(View):
self.view_origin = view_origin self.view_origin = view_origin
self.user_id = False self.user_id = False
self.user_conf: UserConfig = False self.user_conf: UserConfig = False
self.default_conf = False
self.context = False self.context = False
def get_all_view_styles(self): def get_all_view_styles(self):
@ -73,11 +73,10 @@ class ArchivistViewConfig(View):
"""build default context for every view""" """build default context for every view"""
self.user_id = user_id self.user_id = user_id
self.user_conf = UserConfig(self.user_id) self.user_conf = UserConfig(self.user_id)
self.default_conf = AppConfig().config
self.context = { self.context = {
"colors": self.user_conf.get_value("colors"), "colors": self.user_conf.get_value("colors"),
"cast": self.default_conf["application"]["enable_cast"], "cast": EnvironmentSettings.ENABLE_CAST,
"sort_by": self.user_conf.get_value("sort_by"), "sort_by": self.user_conf.get_value("sort_by"),
"sort_order": self.user_conf.get_value("sort_order"), "sort_order": self.user_conf.get_value("sort_order"),
"view_style": self.user_conf.get_value( "view_style": self.user_conf.get_value(
@ -877,7 +876,7 @@ class VideoView(MinView):
"video": video_data, "video": video_data,
"playlist_nav": playlist_nav, "playlist_nav": playlist_nav,
"title": video_data.get("title"), "title": video_data.get("title"),
"cast": config_handler.config["application"]["enable_cast"], "cast": EnvironmentSettings.ENABLE_CAST,
"config": config_handler.config, "config": config_handler.config,
"position": time_parser(request.GET.get("t")), "position": time_parser(request.GET.get("t")),
"reindex": reindex.get("state"), "reindex": reindex.get("state"),

2
tubearchivist/static/script.js
View File

@ -470,7 +470,7 @@ function createPlayer(button) {
// If cast integration is enabled create cast button // If cast integration is enabled create cast button
let castButton = ''; let castButton = '';
if (videoData.config.application.enable_cast) { if (videoData.config.enable_cast) {
castButton = `<google-cast-launcher id="castbutton"></google-cast-launcher>`; castButton = `<google-cast-launcher id="castbutton"></google-cast-launcher>`;
} }