tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move the startup application settings to a new class (#571) 

* Move the startup application settings to a new class

* Replace settings methods with static fields

* Move Redis and ES configuration to the settings class

* Fix environment python imports

* Update envcheck to use the new settings
This commit is contained in:
Clark 2023-10-28 03:27:03 +00:00 committed by GitHub
parent 5165c3e34a
commit 4d111aff82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 153 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tubearchivist/api/src/search_processor.py
View File

@ -7,15 +7,14 @@ Functionality:
import urllib.parse
from home.src.download.thumbnails import ThumbManager
from home.src.ta.config import AppConfig
from home.src.ta.helper import date_praser, get_duration_str
from home.src.ta.settings import EnvironmentSettings
class SearchProcess:
"""process search results"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
def __init__(self, response):
self.response = response

9
tubearchivist/api/views.py
View File

@ -20,6 +20,7 @@ from home.src.index.playlist import YoutubePlaylist
from home.src.index.reindex import ReindexProgress
from home.src.index.video import SponsorBlock, YoutubeVideo
from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskCommand, TaskManager
from home.src.ta.urlparser import Parser
@ -56,7 +57,13 @@ class ApiBaseView(APIView):
def __init__(self):
super().__init__()
self.response = {"data": False, "config": AppConfig().config}
self.response = {
"data": False,
"config": {
"enable_cast": EnvironmentSettings.ENABLE_CAST,
"downloads": AppConfig().config["downloads"],
},
}
self.data = {"query": {"match_all": {}}}
self.status_code = False
self.context = False

19
tubearchivist/config/management/commands/ta_envcheck.py
View File

@ -11,6 +11,7 @@ import re
from django.core.management.base import BaseCommand, CommandError
from home.models import Account
from home.src.ta.settings import EnvironmentSettings
LOGO = """
@ -96,18 +97,14 @@ class Command(BaseCommand):
def _elastic_user_overwrite(self):
"""check for ELASTIC_USER overwrite"""
self.stdout.write("[2] set default ES user")
if not os.environ.get("ELASTIC_USER"):
os.environ.setdefault("ELASTIC_USER", "elastic")
env = os.environ.get("ELASTIC_USER")
self.stdout.write("[2] check ES user overwrite")
env = EnvironmentSettings.ES_USER
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
def _ta_port_overwrite(self):
"""set TA_PORT overwrite for nginx"""
self.stdout.write("[3] check TA_PORT overwrite")
overwrite = os.environ.get("TA_PORT")
overwrite = EnvironmentSettings.TA_PORT
if not overwrite:
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
return
@ -125,7 +122,7 @@ class Command(BaseCommand):
def _ta_uwsgi_overwrite(self):
"""set TA_UWSGI_PORT overwrite"""
self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
overwrite = os.environ.get("TA_UWSGI_PORT")
overwrite = EnvironmentSettings.TA_UWSGI_PORT
if not overwrite:
message = " TA_UWSGI_PORT is not set"
self.stdout.write(self.style.SUCCESS(message))
@ -151,7 +148,7 @@ class Command(BaseCommand):
def _enable_cast_overwrite(self):
"""cast workaround, remove auth for static files in nginx"""
self.stdout.write("[5] check ENABLE_CAST overwrite")
overwrite = os.environ.get("ENABLE_CAST")
overwrite = EnvironmentSettings.ENABLE_CAST
if not overwrite:
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
return
@ -174,8 +171,8 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS(message))
return
name = os.environ.get("TA_USERNAME")
password = os.environ.get("TA_PASSWORD")
name = EnvironmentSettings.TA_USERNAME
password = EnvironmentSettings.TA_PASSWORD
Account.objects.create_superuser(name, password)
message = f" ✓ new superuser with name {name} created"
self.stdout.write(self.style.SUCCESS(message))

9
tubearchivist/config/management/commands/ta_migpath.py
View File

@ -6,8 +6,8 @@ import shutil
from django.core.management.base import BaseCommand
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
TOPIC = """
@ -58,8 +58,7 @@ class FolderMigration:
"""migrate video archive folder"""
def __init__(self):
self.config = AppConfig().config
self.videos = self.config["application"]["videos"]
self.videos = EnvironmentSettings.MEDIA_DIR
self.bulk_list = []
def get_to_migrate(self):
@ -84,8 +83,8 @@ class FolderMigration:
def create_folders(self, to_migrate):
"""create required channel folders"""
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
for channel_id in all_channel_ids:

8
tubearchivist/config/management/commands/ta_startup.py
View File

@ -14,6 +14,7 @@ from home.src.es.snapshot import ElasticSnapshot
from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.helper import clear_dl_cache
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
from home.src.ta.users import UserConfig
@ -69,7 +70,7 @@ class Command(BaseCommand):
"playlists",
"videos",
]
cache_dir = AppConfig().config["application"]["cache_dir"]
cache_dir = EnvironmentSettings.CACHE_DIR
for folder in folders:
folder_path = os.path.join(cache_dir, folder)
os.makedirs(folder_path, exist_ok=True)
@ -119,8 +120,7 @@ class Command(BaseCommand):
def _clear_dl_cache(self):
"""clear leftover files from dl cache"""
self.stdout.write("[5] clear leftover files from dl cache")
config = AppConfig().config
leftover_files = clear_dl_cache(config)
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
if leftover_files:
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
@ -152,7 +152,7 @@ class Command(BaseCommand):
def _mig_set_streams(self):
"""migration: update from 0.3.5 to 0.3.6, set streams and media_size"""
self.stdout.write("[MIGRATION] index streams and media size")
videos = AppConfig().config["application"]["videos"]
videos = EnvironmentSettings.MEDIA_DIR
data = {
"query": {
"bool": {"must_not": [{"exists": {"field": "streams"}}]}

8
tubearchivist/config/settings.py
View File

@ -17,8 +17,8 @@ from pathlib import Path
import ldap
from corsheaders.defaults import default_headers
from django_auth_ldap.config import LDAPSearch
from home.src.ta.config import AppConfig
from home.src.ta.helper import ta_host_parser
from home.src.ta.settings import EnvironmentSettings
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
SECRET_KEY = PW_HASH.hexdigest()
# SECURITY WARNING: don't run with debug turned on in production!
@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")):
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
CACHE_DIR = AppConfig().config["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
DATABASES = {
"default": {
@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
# https://docs.djangoproject.com/en/3.2/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = environ.get("TZ") or "UTC"
TIME_ZONE = EnvironmentSettings.TZ
USE_I18N = True
USE_L10N = True
USE_TZ = True

4
tubearchivist/home/config.json
View File

@ -25,10 +25,6 @@
"integrate_sponsorblock": false
},
"application": {
"app_root": "/app",
"cache_dir": "/cache",
"videos": "/youtube",
"enable_cast": false,
"enable_snapshot": true
},
"scheduler": {

12
tubearchivist/home/src/download/thumbnails.py
View File

@ -11,7 +11,7 @@ from time import sleep
import requests
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.settings import EnvironmentSettings
from mutagen.mp4 import MP4, MP4Cover
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
class ThumbManagerBase:
"""base class for thumbnail management"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
@ -70,7 +69,7 @@ class ThumbManagerBase:
img_raw = Image.open(self.fallback)
return img_raw
app_root = self.CONFIG["application"]["app_root"]
app_root = EnvironmentSettings.APP_DIR
default_map = {
"video": os.path.join(
app_root, "static/img/default-video-thumb.jpg"
@ -380,9 +379,8 @@ class ThumbFilesystem:
class EmbedCallback:
"""callback class to embed thumbnails"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
MEDIA_DIR = CONFIG["application"]["videos"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
FORMAT = MP4Cover.FORMAT_JPEG
def __init__(self, source, index_name, counter=0):

7
tubearchivist/home/src/download/yt_dlp_base.py
View File

@ -10,6 +10,7 @@ from http import cookiejar
from io import StringIO
import yt_dlp
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
@ -86,6 +87,7 @@ class CookieHandler:
def __init__(self, config):
self.cookie_io = False
self.config = config
self.cache_dir = EnvironmentSettings.CACHE_DIR
def get(self):
"""get cookie io stream"""
@ -95,8 +97,9 @@ class CookieHandler:
def import_cookie(self):
"""import cookie from file"""
cache_path = self.config["application"]["cache_dir"]
import_path = os.path.join(cache_path, "import", "cookies.google.txt")
import_path = os.path.join(
self.cache_dir, "import", "cookies.google.txt"
)
try:
with open(import_path, encoding="utf-8") as cookie_file:

24
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video
from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class DownloadPostProcess:
@ -153,6 +154,8 @@ class VideoDownloader:
self.youtube_id_list = youtube_id_list
self.task = task
self.config = AppConfig().config
self.cache_dir = EnvironmentSettings.CACHE_DIR
self.media_dir = EnvironmentSettings.MEDIA_DIR
self._build_obs()
self.channels = set()
self.videos = set()
@ -262,10 +265,7 @@ class VideoDownloader:
"""initial obs"""
self.obs = {
"merge_output_format": "mp4",
"outtmpl": (
self.config["application"]["cache_dir"]
+ "/download/%(id)s.mp4"
),
"outtmpl": (self.cache_dir + "/download/%(id)s.mp4"),
"progress_hooks": [self._progress_hook],
"noprogress": True,
"continuedl": True,
@ -340,7 +340,7 @@ class VideoDownloader:
if format_overwrite:
obs["format"] = format_overwrite
dl_cache = self.config["application"]["cache_dir"] + "/download/"
dl_cache = self.cache_dir + "/download/"
# check if already in cache to continue from there
all_cached = ignore_filelist(os.listdir(dl_cache))
@ -370,20 +370,20 @@ class VideoDownloader:
def move_to_archive(self, vid_dict):
"""move downloaded video from cache to archive"""
videos = self.config["application"]["videos"]
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
# make folder
folder = os.path.join(videos, vid_dict["channel"]["channel_id"])
folder = os.path.join(
self.media_dir, vid_dict["channel"]["channel_id"]
)
if not os.path.exists(folder):
os.makedirs(folder)
if host_uid and host_gid:
os.chown(folder, host_uid, host_gid)
# move media file
media_file = vid_dict["youtube_id"] + ".mp4"
cache_dir = self.config["application"]["cache_dir"]
old_path = os.path.join(cache_dir, "download", media_file)
new_path = os.path.join(videos, vid_dict["media_url"])
old_path = os.path.join(self.cache_dir, "download", media_file)
new_path = os.path.join(self.media_dir, vid_dict["media_url"])
# move media file and fix permission
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
if host_uid and host_gid:

7
tubearchivist/home/src/es/backup.py
View File

@ -13,6 +13,7 @@ from datetime import datetime
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import get_mapping, ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class ElasticBackup:
@ -22,7 +23,7 @@ class ElasticBackup:
def __init__(self, reason=False, task=False):
self.config = AppConfig().config
self.cache_dir = self.config["application"]["cache_dir"]
self.cache_dir = EnvironmentSettings.CACHE_DIR
self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = get_mapping()
self.reason = reason
@ -217,6 +218,7 @@ class BackupCallback:
self.index_name = index_name
self.counter = counter
self.timestamp = datetime.now().strftime("%Y%m%d")
self.cache_dir = EnvironmentSettings.CACHE_DIR
def run(self):
"""run the junk task"""
@ -243,9 +245,8 @@ class BackupCallback:
def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk"""
cache_dir = AppConfig().config["application"]["cache_dir"]
index = self.index_name.lstrip("ta_")
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
file_path = os.path.join(cache_dir, "backup", file_name)
file_path = os.path.join(self.cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content)

24
tubearchivist/home/src/es/connect.py
View File

@ -6,11 +6,11 @@ functionality:
# pylint: disable=missing-timeout
import json
import os
from typing import Any
import requests
import urllib3
from home.src.ta.settings import EnvironmentSettings
class ElasticWrap:
@ -18,16 +18,14 @@ class ElasticWrap:
returns response json and status code tuple
"""
ES_URL: str = str(os.environ.get("ES_URL"))
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))
def __init__(self, path: str):
self.url: str = f"{self.ES_URL}/{path}"
self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS)
self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
self.auth: tuple[str, str] = (
EnvironmentSettings.ES_USER,
EnvironmentSettings.ES_PASS,
)
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def get(
@ -43,7 +41,7 @@ class ElasticWrap:
"timeout": timeout,
}
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
if data:
@ -78,7 +76,7 @@ class ElasticWrap:
}
)
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.post(self.url, **kwargs)
@ -103,7 +101,7 @@ class ElasticWrap:
"auth": self.auth,
}
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.put(self.url, **kwargs)
@ -130,7 +128,7 @@ class ElasticWrap:
if data:
kwargs["json"] = data
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.delete(self.url, **kwargs)

3
tubearchivist/home/src/es/snapshot.py
View File

@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo
from home.src.es.connect import ElasticWrap
from home.src.ta.helper import get_mapping
from home.src.ta.settings import EnvironmentSettings
class ElasticSnapshot:
@ -256,7 +257,7 @@ class ElasticSnapshot:
expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
date = datetime.strptime(date_utc, expected_format)
local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
converted_str = converted.strftime("%Y-%m-%d %H:%M")
return converted_str

5
tubearchivist/home/src/index/channel.py
View File

@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.generic import YouTubeItem
from home.src.index.playlist import YoutubePlaylist
from home.src.ta.settings import EnvironmentSettings
class YoutubeChannel(YouTubeItem):
@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem):
def _info_json_fallback(self):
"""read channel info.json for additional metadata"""
info_json = os.path.join(
self.config["application"]["cache_dir"],
EnvironmentSettings.CACHE_DIR,
"import",
f"{self.youtube_id}.info.json",
)
@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem):
def get_folder_path(self):
"""get folder where media files get stored"""
folder_path = os.path.join(
self.app_conf["videos"],
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel_id"],
)
return folder_path

4
tubearchivist/home/src/index/filesystem.py
View File

@ -8,14 +8,14 @@ import os
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class Scanner:
"""scan index and filesystem"""
VIDEOS: str = AppConfig().config["application"]["videos"]
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
def __init__(self, task=False) -> None:
self.task = task

1
tubearchivist/home/src/index/generic.py
View File

@ -26,7 +26,6 @@ class YouTubeItem:
self.youtube_id = youtube_id
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
self.config = AppConfig().config
self.app_conf = self.config["application"]
self.youtube_meta = False
self.json_data = False

11
tubearchivist/home/src/index/manual.py
View File

@ -16,6 +16,7 @@ from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
from PIL import Image
from yt_dlp.utils import ISO639Utils
@ -28,7 +29,7 @@ class ImportFolderScanner:
"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
"""All extensions should be in lowercase until better handling is in place.
@ -433,9 +434,9 @@ class ManualImport:
def _move_to_archive(self, json_data):
"""move identified media file to archive"""
videos = self.config["application"]["videos"]
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
videos = EnvironmentSettings.MEDIA_DIR
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel)
@ -472,7 +473,7 @@ class ManualImport:
os.remove(subtitle_file)
channel_info = os.path.join(
self.config["application"]["cache_dir"],
EnvironmentSettings.CACHE_DIR,
"import",
f"{json_data['channel']['channel_id']}.info.json",
)

5
tubearchivist/home/src/index/reindex.py
View File

@ -19,6 +19,7 @@ from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisQueue
@ -293,7 +294,7 @@ class Reindex(ReindexBase):
# get new
media_url = os.path.join(
self.config["application"]["videos"], es_meta["media_url"]
EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
)
video.build_json(media_path=media_url)
if not video.youtube_meta:
@ -328,7 +329,7 @@ class Reindex(ReindexBase):
def _rename_media_file(self, media_url_is, media_url_should):
"""handle title change"""
print(f"[reindex] fix media_url {media_url_is} to {media_url_should}")
videos = self.config["application"]["videos"]
videos = EnvironmentSettings.MEDIA_DIR
old_path = os.path.join(videos, media_url_is)
new_path = os.path.join(videos, media_url_should)
os.rename(old_path, new_path)

9
tubearchivist/home/src/index/subtitle.py
View File

@ -12,6 +12,7 @@ from datetime import datetime
import requests
from home.src.es.connect import ElasticWrap
from home.src.ta.helper import requests_headers
from home.src.ta.settings import EnvironmentSettings
class YoutubeSubtitle:
@ -113,7 +114,7 @@ class YoutubeSubtitle:
def download_subtitles(self, relevant_subtitles):
"""download subtitle files to archive"""
videos_base = self.video.config["application"]["videos"]
videos_base = EnvironmentSettings.MEDIA_DIR
indexed = []
for subtitle in relevant_subtitles:
dest_path = os.path.join(videos_base, subtitle["media_url"])
@ -149,8 +150,8 @@ class YoutubeSubtitle:
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(subtitle_str)
host_uid = self.video.config["application"]["HOST_UID"]
host_gid = self.video.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
if host_uid and host_gid:
os.chown(dest_path, host_uid, host_gid)
@ -162,7 +163,7 @@ class YoutubeSubtitle:
def delete(self, subtitles=False):
"""delete subtitles from index and filesystem"""
youtube_id = self.video.youtube_id
videos_base = self.video.config["application"]["videos"]
videos_base = EnvironmentSettings.MEDIA_DIR
# delete files
if subtitles:
files = [i["media_url"] for i in subtitles]

7
tubearchivist/home/src/index/video.py
View File

@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle
from home.src.index.video_constants import VideoTypeEnum
from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.users import UserConfig
from ryd_client import ryd_client
@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
def build_dl_cache_path(self):
"""find video path in dl cache"""
cache_dir = self.app_conf["cache_dir"]
cache_dir = EnvironmentSettings.CACHE_DIR
video_id = self.json_data["youtube_id"]
cache_path = f"{cache_dir}/download/{video_id}.mp4"
if os.path.exists(cache_path):
return cache_path
channel_path = os.path.join(
self.app_conf["videos"],
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel"]["channel_id"],
f"{video_id}.mp4",
)
@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
if not self.json_data:
raise FileNotFoundError
video_base = self.app_conf["videos"]
video_base = EnvironmentSettings.MEDIA_DIR
media_url = self.json_data.get("media_url")
file_path = os.path.join(video_base, media_url)
try:

19
tubearchivist/home/src/ta/config.py
View File

@ -5,7 +5,6 @@ Functionality:
"""
import json
import os
import re
from random import randint
from time import sleep
@ -28,7 +27,6 @@ class AppConfig:
if not config:
config = self.get_config_file()
config["application"].update(self.get_config_env())
return config
def get_config_file(self):
@ -36,25 +34,8 @@ class AppConfig:
with open("home/config.json", "r", encoding="utf-8") as f:
config_file = json.load(f)
config_file["application"].update(self.get_config_env())
return config_file
@staticmethod
def get_config_env():
"""read environment application variables.
Connection to ES is managed in ElasticWrap and the
connection to Redis is managed in RedisArchivist."""
application = {
"HOST_UID": int(os.environ.get("HOST_UID", False)),
"HOST_GID": int(os.environ.get("HOST_GID", False)),
"enable_cast": bool(os.environ.get("ENABLE_CAST")),
}
return application
@staticmethod
def get_config_redis():
"""read config json set from redis to overwrite defaults"""

8
tubearchivist/home/src/ta/helper.py
View File

@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float:
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
def clear_dl_cache(config: dict) -> int:
def clear_dl_cache(cache_dir: str) -> int:
"""clear leftover files from dl cache"""
print("clear download cache")
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
leftover_files = ignore_filelist(os.listdir(cache_dir))
download_cache_dir = os.path.join(cache_dir, "download")
leftover_files = ignore_filelist(os.listdir(download_cache_dir))
for cached in leftover_files:
to_delete = os.path.join(cache_dir, cached)
to_delete = os.path.join(download_cache_dir, cached)
os.remove(to_delete)
return len(leftover_files)

39
tubearchivist/home/src/ta/settings.py Normal file
View File

@ -0,0 +1,39 @@
"""
Functionality:
- read and write application config backed by ES
- encapsulate persistence of application properties
"""
import os
class EnvironmentSettings:
"""
Handle settings for the application that are driven from the environment.
These will not change when the user is using the application.
These settings are only provided only on startup.
"""
HOST_UID: int = int(os.environ.get("HOST_UID", False))
HOST_GID: int = int(os.environ.get("HOST_GID", False))
ENABLE_CAST: bool = bool(os.environ.get("ENABLE_CAST"))
TZ: str = str(os.environ.get("TZ", "UTC"))
TA_PORT: int = int(os.environ.get("TA_PORT", False))
TA_UWSGI_PORT: int = int(os.environ.get("TA_UWSGI_PORT", False))
TA_USERNAME: str = str(os.environ.get("TA_USERNAME"))
TA_PASSWORD: str = str(os.environ.get("TA_PASSWORD"))
# Application Paths
MEDIA_DIR: str = str(os.environ.get("TA_MEDIA_DIR", "/youtube"))
APP_DIR: str = str(os.environ.get("TA_APP_DIR", "/app"))
CACHE_DIR: str = str(os.environ.get("TA_CACHE_DIR", "/cache"))
# Redis
REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
REDIS_PORT: int = int(os.environ.get("REDIS_PORT", 6379))
REDIS_NAME_SPACE: str = str(os.environ.get("REDIS_NAME_SPACE", "ta:"))
# ElasticSearch
ES_URL: str = str(os.environ.get("ES_URL"))
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(os.environ.get("ELASTIC_USER", "elastic"))
ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))

11
tubearchivist/home/src/ta/ta_redis.py
View File

@ -6,20 +6,21 @@ functionality:
"""
import json
import os
import redis
from home.src.ta.settings import EnvironmentSettings
class RedisBase:
"""connection base for redis"""
REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
NAME_SPACE: str = "ta:"
NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
def __init__(self):
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
self.conn = redis.Redis(
host=EnvironmentSettings.REDIS_HOST,
port=EnvironmentSettings.REDIS_PORT,
)
class RedisArchivist(RedisBase):

7
tubearchivist/home/src/ta/users.py
View File

@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False):
class UserConfig:
"""Handle settings for an individual user
Create getters and setters for usage in the application.
Although tedious it helps prevents everything caring about how properties
are persisted. Plus it allows us to save anytime any value is set.
"""
"""Handle settings for an individual user"""
_DEFAULT_USER_SETTINGS = UserConfigType(
colors="dark",

11
tubearchivist/home/tasks.py
View File

@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.notify import Notifications
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
from home.src.ta.urlparser import Parser
CONFIG = AppConfig().config
REDIS_HOST = os.environ.get("REDIS_HOST")
REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
REDIS_HOST = EnvironmentSettings.REDIS_HOST
REDIS_PORT = EnvironmentSettings.REDIS_PORT
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
app = Celery(
@ -39,9 +40,11 @@ app = Celery(
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
result_extended=True,
)
app.config_from_object("django.conf:settings", namespace="ta:")
app.config_from_object(
"django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
)
app.autodiscover_tasks()
app.conf.timezone = os.environ.get("TZ") or "UTC"
app.conf.timezone = EnvironmentSettings.TZ
class BaseTask(Task):

7
tubearchivist/home/views.py
View File

@ -42,6 +42,7 @@ from home.src.index.reindex import ReindexProgress
from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.helper import time_parser
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.users import UserConfig
from home.tasks import index_channel_playlists, subscribe_to
@ -56,7 +57,6 @@ class ArchivistViewConfig(View):
self.view_origin = view_origin
self.user_id = False
self.user_conf: UserConfig = False
self.default_conf = False
self.context = False
def get_all_view_styles(self):
@ -73,11 +73,10 @@ class ArchivistViewConfig(View):
"""build default context for every view"""
self.user_id = user_id
self.user_conf = UserConfig(self.user_id)
self.default_conf = AppConfig().config
self.context = {
"colors": self.user_conf.get_value("colors"),
"cast": self.default_conf["application"]["enable_cast"],
"cast": EnvironmentSettings.ENABLE_CAST,
"sort_by": self.user_conf.get_value("sort_by"),
"sort_order": self.user_conf.get_value("sort_order"),
"view_style": self.user_conf.get_value(
@ -877,7 +876,7 @@ class VideoView(MinView):
"video": video_data,
"playlist_nav": playlist_nav,
"title": video_data.get("title"),
"cast": config_handler.config["application"]["enable_cast"],
"cast": EnvironmentSettings.ENABLE_CAST,
"config": config_handler.config,
"position": time_parser(request.GET.get("t")),
"reindex": reindex.get("state"),

2
tubearchivist/static/script.js
View File

@ -470,7 +470,7 @@ function createPlayer(button) {
// If cast integration is enabled create cast button
let castButton = '';
if (videoData.config.application.enable_cast) {
if (videoData.config.enable_cast) {
castButton = `<google-cast-launcher id="castbutton"></google-cast-launcher>`;
}