tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Read only user roles, refac env var builder, #build 

Changed:
- Added view only user role
- Fixed media download URL builder
- Changed environment settings builder away from redis
- Improved dashboard
This commit is contained in:
Simon 2023-11-01 09:24:21 +07:00
parent 7236bea29a 58ea256b44
commit 6892cbbc19
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
43 changed files with 548 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tubearchivist/api/src/aggs.py
View File

@ -117,7 +117,7 @@ class WatchProgress(AggBase):
all_duration = int(aggregations["total_duration"].get("value"))
response.update(
{
"all": {
"total": {
"duration": all_duration,
"duration_str": get_duration_str(all_duration),
"items": aggregations["total_vids"].get("value"),

5
tubearchivist/api/src/search_processor.py
View File

@ -7,15 +7,14 @@ Functionality:
import urllib.parse
from home.src.download.thumbnails import ThumbManager
from home.src.ta.config import AppConfig
from home.src.ta.helper import date_praser, get_duration_str
from home.src.ta.settings import EnvironmentSettings
class SearchProcess:
"""process search results"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
def __init__(self, response):
self.response = response

58
tubearchivist/api/views.py
View File

@ -18,6 +18,7 @@ from home.src.index.playlist import YoutubePlaylist
from home.src.index.reindex import ReindexProgress
from home.src.index.video import SponsorBlock, YoutubeVideo
from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskCommand, TaskManager
from home.src.ta.urlparser import Parser
@ -28,28 +29,56 @@ from home.tasks import (
extrac_dl,
subscribe_to,
)
from rest_framework import permissions
from rest_framework.authentication import (
SessionAuthentication,
TokenAuthentication,
)
from rest_framework.authtoken.models import Token
from rest_framework.authtoken.views import ObtainAuthToken
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView
def check_admin(user):
"""check for admin permission for restricted views"""
return user.is_staff or user.groups.filter(name="admin").exists()
class AdminOnly(permissions.BasePermission):
"""allow only admin"""
def has_permission(self, request, view):
return check_admin(request.user)
class AdminWriteOnly(permissions.BasePermission):
"""allow only admin writes"""
def has_permission(self, request, view):
if request.method in permissions.SAFE_METHODS:
return permissions.IsAuthenticated().has_permission(request, view)
return check_admin(request.user)
class ApiBaseView(APIView):
"""base view to inherit from"""
authentication_classes = [SessionAuthentication, TokenAuthentication]
permission_classes = [IsAuthenticated]
permission_classes = [permissions.IsAuthenticated]
search_base = ""
data = ""
def __init__(self):
super().__init__()
self.response = {"data": False, "config": AppConfig().config}
self.response = {
"data": False,
"config": {
"enable_cast": EnvironmentSettings.ENABLE_CAST,
"downloads": AppConfig().config["downloads"],
},
}
self.data = {"query": {"match_all": {}}}
self.status_code = False
self.context = False
@ -102,6 +131,7 @@ class VideoApiView(ApiBaseView):
"""
search_base = "ta_video/_doc/"
permission_classes = [AdminWriteOnly]
def get(self, request, video_id):
# pylint: disable=unused-argument
@ -165,7 +195,6 @@ class VideoProgressView(ApiBaseView):
message = {"position": position, "youtube_id": video_id}
RedisArchivist().set_message(key, message)
self.response = request.data
return Response(self.response)
def delete(self, request, video_id):
@ -276,6 +305,7 @@ class ChannelApiView(ApiBaseView):
"""
search_base = "ta_channel/_doc/"
permission_classes = [AdminWriteOnly]
def get(self, request, channel_id):
# pylint: disable=unused-argument
@ -306,6 +336,7 @@ class ChannelApiListView(ApiBaseView):
search_base = "ta_channel/_search/"
valid_filter = ["subscribed"]
permission_classes = [AdminWriteOnly]
def get(self, request):
"""get request"""
@ -419,6 +450,7 @@ class PlaylistApiListView(ApiBaseView):
"""
search_base = "ta_playlist/_search/"
permission_classes = [AdminWriteOnly]
def get(self, request):
"""handle get request"""
@ -467,6 +499,7 @@ class PlaylistApiView(ApiBaseView):
"""
search_base = "ta_playlist/_doc/"
permission_classes = [AdminWriteOnly]
def get(self, request, playlist_id):
# pylint: disable=unused-argument
@ -513,6 +546,7 @@ class DownloadApiView(ApiBaseView):
search_base = "ta_download/_doc/"
valid_status = ["pending", "ignore", "priority"]
permission_classes = [AdminOnly]
def get(self, request, video_id):
# pylint: disable=unused-argument
@ -559,6 +593,7 @@ class DownloadApiListView(ApiBaseView):
search_base = "ta_download/_search/"
valid_filter = ["pending", "ignore"]
permission_classes = [AdminOnly]
def get(self, request):
"""get request"""
@ -667,6 +702,8 @@ class SnapshotApiListView(ApiBaseView):
POST: take snapshot now
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
@ -691,6 +728,8 @@ class SnapshotApiView(ApiBaseView):
DELETE: delete a snapshot
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request, snapshot_id):
"""handle get request"""
@ -730,6 +769,8 @@ class TaskListView(ApiBaseView):
GET: return a list of all stored task results
"""
permission_classes = [AdminOnly]
def get(self, request):
"""handle get request"""
# pylint: disable=unused-argument
@ -744,6 +785,8 @@ class TaskNameListView(ApiBaseView):
POST: start new background process
"""
permission_classes = [AdminOnly]
def get(self, request, task_name):
"""handle get request"""
# pylint: disable=unused-argument
@ -782,6 +825,7 @@ class TaskIDView(ApiBaseView):
"""
valid_commands = ["stop", "kill"]
permission_classes = [AdminOnly]
def get(self, request, task_id):
"""handle get request"""
@ -833,6 +877,8 @@ class RefreshView(ApiBaseView):
POST: start a manual refresh task
"""
permission_classes = [AdminOnly]
def get(self, request):
"""handle get request"""
request_type = request.GET.get("type")
@ -866,6 +912,8 @@ class CookieView(ApiBaseView):
PUT: import cookie
"""
permission_classes = [AdminOnly]
@staticmethod
def get(request):
"""handle get request"""
@ -953,6 +1001,8 @@ class TokenView(ApiBaseView):
DELETE: revoke the token
"""
permission_classes = [AdminOnly]
@staticmethod
def delete(request):
print("revoke API token")

8
tubearchivist/config/management/commands/ta_connection.py
View File

@ -3,12 +3,12 @@ Functionality:
- check that all connections are working
"""
from os import environ
from time import sleep
import requests
from django.core.management.base import BaseCommand, CommandError
from home.src.es.connect import ElasticWrap
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
TOPIC = """
@ -156,10 +156,8 @@ class Command(BaseCommand):
" 🗙 path.repo env var not found. "
+ "set the following env var to the ES container:\n"
+ " path.repo="
+ environ.get(
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
),
+ EnvironmentSettings.ES_SNAPSHOT_DIR
)
self.stdout.write(self.style.ERROR(f"{message}"))
self.stdout.write(self.style.ERROR(message))
sleep(60)
raise CommandError(message)

19
tubearchivist/config/management/commands/ta_envcheck.py
View File

@ -11,6 +11,7 @@ import re
from django.core.management.base import BaseCommand, CommandError
from home.models import Account
from home.src.ta.settings import EnvironmentSettings
LOGO = """
@ -96,18 +97,14 @@ class Command(BaseCommand):
def _elastic_user_overwrite(self):
"""check for ELASTIC_USER overwrite"""
self.stdout.write("[2] set default ES user")
if not os.environ.get("ELASTIC_USER"):
os.environ.setdefault("ELASTIC_USER", "elastic")
env = os.environ.get("ELASTIC_USER")
self.stdout.write("[2] check ES user overwrite")
env = EnvironmentSettings.ES_USER
self.stdout.write(self.style.SUCCESS(f" ✓ ES user is set to {env}"))
def _ta_port_overwrite(self):
"""set TA_PORT overwrite for nginx"""
self.stdout.write("[3] check TA_PORT overwrite")
overwrite = os.environ.get("TA_PORT")
overwrite = EnvironmentSettings.TA_PORT
if not overwrite:
self.stdout.write(self.style.SUCCESS(" TA_PORT is not set"))
return
@ -125,7 +122,7 @@ class Command(BaseCommand):
def _ta_uwsgi_overwrite(self):
"""set TA_UWSGI_PORT overwrite"""
self.stdout.write("[4] check TA_UWSGI_PORT overwrite")
overwrite = os.environ.get("TA_UWSGI_PORT")
overwrite = EnvironmentSettings.TA_UWSGI_PORT
if not overwrite:
message = " TA_UWSGI_PORT is not set"
self.stdout.write(self.style.SUCCESS(message))
@ -151,7 +148,7 @@ class Command(BaseCommand):
def _enable_cast_overwrite(self):
"""cast workaround, remove auth for static files in nginx"""
self.stdout.write("[5] check ENABLE_CAST overwrite")
overwrite = os.environ.get("ENABLE_CAST")
overwrite = EnvironmentSettings.ENABLE_CAST
if not overwrite:
self.stdout.write(self.style.SUCCESS(" ENABLE_CAST is not set"))
return
@ -174,8 +171,8 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS(message))
return
name = os.environ.get("TA_USERNAME")
password = os.environ.get("TA_PASSWORD")
name = EnvironmentSettings.TA_USERNAME
password = EnvironmentSettings.TA_PASSWORD
Account.objects.create_superuser(name, password)
message = f" ✓ new superuser with name {name} created"
self.stdout.write(self.style.SUCCESS(message))

9
tubearchivist/config/management/commands/ta_migpath.py
View File

@ -6,8 +6,8 @@ import shutil
from django.core.management.base import BaseCommand
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
TOPIC = """
@ -58,8 +58,7 @@ class FolderMigration:
"""migrate video archive folder"""
def __init__(self):
self.config = AppConfig().config
self.videos = self.config["application"]["videos"]
self.videos = EnvironmentSettings.MEDIA_DIR
self.bulk_list = []
def get_to_migrate(self):
@ -84,8 +83,8 @@ class FolderMigration:
def create_folders(self, to_migrate):
"""create required channel folders"""
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
all_channel_ids = {i["channel"]["channel_id"] for i in to_migrate}
for channel_id in all_channel_ids:

89
tubearchivist/config/management/commands/ta_startup.py
View File

@ -8,12 +8,11 @@ import os
from time import sleep
from django.core.management.base import BaseCommand, CommandError
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.es.index_setup import ElasitIndexWrap
from home.src.es.snapshot import ElasticSnapshot
from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.config import AppConfig, ReleaseVersion
from home.src.ta.helper import clear_dl_cache
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
from home.src.ta.users import UserConfig
@ -43,8 +42,6 @@ class Command(BaseCommand):
self._version_check()
self._mig_index_setup()
self._mig_snapshot_check()
self._mig_set_streams()
self._mig_set_autostart()
self._mig_move_users_to_es()
def _sync_redis_state(self):
@ -69,7 +66,7 @@ class Command(BaseCommand):
"playlists",
"videos",
]
cache_dir = AppConfig().config["application"]["cache_dir"]
cache_dir = EnvironmentSettings.CACHE_DIR
for folder in folders:
folder_path = os.path.join(cache_dir, folder)
os.makedirs(folder_path, exist_ok=True)
@ -119,8 +116,7 @@ class Command(BaseCommand):
def _clear_dl_cache(self):
"""clear leftover files from dl cache"""
self.stdout.write("[5] clear leftover files from dl cache")
config = AppConfig().config
leftover_files = clear_dl_cache(config)
leftover_files = clear_dl_cache(EnvironmentSettings.CACHE_DIR)
if leftover_files:
self.stdout.write(
self.style.SUCCESS(f" ✓ cleared {leftover_files} files")
@ -149,79 +145,6 @@ class Command(BaseCommand):
self.stdout.write("[MIGRATION] setup snapshots")
ElasticSnapshot().setup()
def _mig_set_streams(self):
"""migration: update from 0.3.5 to 0.3.6, set streams and media_size"""
self.stdout.write("[MIGRATION] index streams and media size")
videos = AppConfig().config["application"]["videos"]
data = {
"query": {
"bool": {"must_not": [{"exists": {"field": "streams"}}]}
},
"_source": ["media_url", "youtube_id"],
}
all_missing = IndexPaginate("ta_video", data).get_results()
if not all_missing:
self.stdout.write(" no videos need updating")
return
total = len(all_missing)
for idx, missing in enumerate(all_missing):
media_url = missing["media_url"]
youtube_id = missing["youtube_id"]
media_path = os.path.join(videos, media_url)
if not os.path.exists(media_path):
self.stdout.write(f" file not found: {media_path}")
self.stdout.write(" run file system rescan to fix")
continue
media = MediaStreamExtractor(media_path)
vid_data = {
"doc": {
"streams": media.extract_metadata(),
"media_size": media.get_file_size(),
}
}
path = f"ta_video/_update/{youtube_id}"
response, status_code = ElasticWrap(path).post(data=vid_data)
if not status_code == 200:
self.stdout.errors(
f" update failed: {path}, {response}, {status_code}"
)
if idx % 100 == 0:
self.stdout.write(f" progress {idx}/{total}")
def _mig_set_autostart(self):
"""migration: update from 0.3.5 to 0.3.6 set auto_start to false"""
self.stdout.write("[MIGRATION] set default download auto_start")
data = {
"query": {
"bool": {"must_not": [{"exists": {"field": "auto_start"}}]}
},
"script": {"source": "ctx._source['auto_start'] = false"},
}
path = "ta_download/_update_by_query"
response, status_code = ElasticWrap(path).post(data=data)
if status_code == 200:
updated = response.get("updated", 0)
if updated:
self.stdout.write(
self.style.SUCCESS(
f" ✓ {updated} videos updated in ta_download"
)
)
else:
self.stdout.write(
" no videos needed updating in ta_download"
)
return
message = " 🗙 ta_download auto_start update failed"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(response)
sleep(60)
raise CommandError(message)
def _mig_move_users_to_es(self): # noqa: C901
"""migration: update from 0.4.1 to 0.4.2 move user config to ES"""
self.stdout.write("[MIGRATION] move user configuration to ES")
@ -299,12 +222,12 @@ class Command(BaseCommand):
f" ✓ Settings for user '{user}' migrated to ES"
)
)
except Exception as e:
except Exception as err:
message = " 🗙 user migration to ES failed"
self.stdout.write(self.style.ERROR(message))
self.stdout.write(self.style.ERROR(e))
self.stdout.write(self.style.ERROR(err))
sleep(60)
raise CommandError(message)
raise CommandError(message) from err
else:
self.stdout.write(
self.style.SUCCESS(

10
tubearchivist/config/settings.py
View File

@ -17,8 +17,8 @@ from pathlib import Path
import ldap
from corsheaders.defaults import default_headers
from django_auth_ldap.config import LDAPSearch
from home.src.ta.config import AppConfig
from home.src.ta.helper import ta_host_parser
from home.src.ta.settings import EnvironmentSettings
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
@ -27,7 +27,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
PW_HASH = hashlib.sha256(environ["TA_PASSWORD"].encode())
PW_HASH = hashlib.sha256(EnvironmentSettings.TA_PASSWORD.encode())
SECRET_KEY = PW_HASH.hexdigest()
# SECURITY WARNING: don't run with debug turned on in production!
@ -180,7 +180,7 @@ if bool(environ.get("TA_LDAP")):
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
CACHE_DIR = AppConfig().config["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
DB_PATH = path.join(CACHE_DIR, "db.sqlite3")
DATABASES = {
"default": {
@ -228,7 +228,7 @@ if bool(environ.get("TA_ENABLE_AUTH_PROXY")):
# https://docs.djangoproject.com/en/3.2/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = environ.get("TZ") or "UTC"
TIME_ZONE = EnvironmentSettings.TZ
USE_I18N = True
USE_L10N = True
USE_TZ = True
@ -269,4 +269,4 @@ CORS_ALLOW_HEADERS = list(default_headers) + [
# TA application settings
TA_UPSTREAM = "https://github.com/tubearchivist/tubearchivist"
TA_VERSION = "v0.4.2"
TA_VERSION = "v0.4.3-unstable"

4
tubearchivist/home/config.json
View File

@ -25,10 +25,6 @@
"integrate_sponsorblock": false
},
"application": {
"app_root": "/app",
"cache_dir": "/cache",
"videos": "/youtube",
"enable_cast": false,
"enable_snapshot": true
},
"scheduler": {

12
tubearchivist/home/src/download/thumbnails.py
View File

@ -11,7 +11,7 @@ from time import sleep
import requests
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.settings import EnvironmentSettings
from mutagen.mp4 import MP4, MP4Cover
from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
@ -21,8 +21,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
class ThumbManagerBase:
"""base class for thumbnail management"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
VIDEO_DIR = os.path.join(CACHE_DIR, "videos")
CHANNEL_DIR = os.path.join(CACHE_DIR, "channels")
PLAYLIST_DIR = os.path.join(CACHE_DIR, "playlists")
@ -70,7 +69,7 @@ class ThumbManagerBase:
img_raw = Image.open(self.fallback)
return img_raw
app_root = self.CONFIG["application"]["app_root"]
app_root = EnvironmentSettings.APP_DIR
default_map = {
"video": os.path.join(
app_root, "static/img/default-video-thumb.jpg"
@ -380,9 +379,8 @@ class ThumbFilesystem:
class EmbedCallback:
"""callback class to embed thumbnails"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
MEDIA_DIR = CONFIG["application"]["videos"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
MEDIA_DIR = EnvironmentSettings.MEDIA_DIR
FORMAT = MP4Cover.FORMAT_JPEG
def __init__(self, source, index_name, counter=0):

7
tubearchivist/home/src/download/yt_dlp_base.py
View File

@ -10,6 +10,7 @@ from http import cookiejar
from io import StringIO
import yt_dlp
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
@ -86,6 +87,7 @@ class CookieHandler:
def __init__(self, config):
self.cookie_io = False
self.config = config
self.cache_dir = EnvironmentSettings.CACHE_DIR
def get(self):
"""get cookie io stream"""
@ -95,8 +97,9 @@ class CookieHandler:
def import_cookie(self):
"""import cookie from file"""
cache_path = self.config["application"]["cache_dir"]
import_path = os.path.join(cache_path, "import", "cookies.google.txt")
import_path = os.path.join(
self.cache_dir, "import", "cookies.google.txt"
)
try:
with open(import_path, encoding="utf-8") as cookie_file:

24
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -21,6 +21,7 @@ from home.src.index.video import YoutubeVideo, index_new_video
from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class DownloadPostProcess:
@ -153,6 +154,8 @@ class VideoDownloader:
self.youtube_id_list = youtube_id_list
self.task = task
self.config = AppConfig().config
self.cache_dir = EnvironmentSettings.CACHE_DIR
self.media_dir = EnvironmentSettings.MEDIA_DIR
self._build_obs()
self.channels = set()
self.videos = set()
@ -262,10 +265,7 @@ class VideoDownloader:
"""initial obs"""
self.obs = {
"merge_output_format": "mp4",
"outtmpl": (
self.config["application"]["cache_dir"]
+ "/download/%(id)s.mp4"
),
"outtmpl": (self.cache_dir + "/download/%(id)s.mp4"),
"progress_hooks": [self._progress_hook],
"noprogress": True,
"continuedl": True,
@ -340,7 +340,7 @@ class VideoDownloader:
if format_overwrite:
obs["format"] = format_overwrite
dl_cache = self.config["application"]["cache_dir"] + "/download/"
dl_cache = self.cache_dir + "/download/"
# check if already in cache to continue from there
all_cached = ignore_filelist(os.listdir(dl_cache))
@ -370,20 +370,20 @@ class VideoDownloader:
def move_to_archive(self, vid_dict):
"""move downloaded video from cache to archive"""
videos = self.config["application"]["videos"]
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
# make folder
folder = os.path.join(videos, vid_dict["channel"]["channel_id"])
folder = os.path.join(
self.media_dir, vid_dict["channel"]["channel_id"]
)
if not os.path.exists(folder):
os.makedirs(folder)
if host_uid and host_gid:
os.chown(folder, host_uid, host_gid)
# move media file
media_file = vid_dict["youtube_id"] + ".mp4"
cache_dir = self.config["application"]["cache_dir"]
old_path = os.path.join(cache_dir, "download", media_file)
new_path = os.path.join(videos, vid_dict["media_url"])
old_path = os.path.join(self.cache_dir, "download", media_file)
new_path = os.path.join(self.media_dir, vid_dict["media_url"])
# move media file and fix permission
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
if host_uid and host_gid:

7
tubearchivist/home/src/es/backup.py
View File

@ -13,6 +13,7 @@ from datetime import datetime
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.ta.config import AppConfig
from home.src.ta.helper import get_mapping, ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class ElasticBackup:
@ -22,7 +23,7 @@ class ElasticBackup:
def __init__(self, reason=False, task=False):
self.config = AppConfig().config
self.cache_dir = self.config["application"]["cache_dir"]
self.cache_dir = EnvironmentSettings.CACHE_DIR
self.timestamp = datetime.now().strftime("%Y%m%d")
self.index_config = get_mapping()
self.reason = reason
@ -217,6 +218,7 @@ class BackupCallback:
self.index_name = index_name
self.counter = counter
self.timestamp = datetime.now().strftime("%Y%m%d")
self.cache_dir = EnvironmentSettings.CACHE_DIR
def run(self):
"""run the junk task"""
@ -243,9 +245,8 @@ class BackupCallback:
def _write_es_json(self, file_content):
"""write nd-json file for es _bulk API to disk"""
cache_dir = AppConfig().config["application"]["cache_dir"]
index = self.index_name.lstrip("ta_")
file_name = f"es_{index}-{self.timestamp}-{self.counter}.json"
file_path = os.path.join(cache_dir, "backup", file_name)
file_path = os.path.join(self.cache_dir, "backup", file_name)
with open(file_path, "a+", encoding="utf-8") as f:
f.write(file_content)

24
tubearchivist/home/src/es/connect.py
View File

@ -6,11 +6,11 @@ functionality:
# pylint: disable=missing-timeout
import json
import os
from typing import Any
import requests
import urllib3
from home.src.ta.settings import EnvironmentSettings
class ElasticWrap:
@ -18,16 +18,14 @@ class ElasticWrap:
returns response json and status code tuple
"""
ES_URL: str = str(os.environ.get("ES_URL"))
ES_PASS: str = str(os.environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(os.environ.get("ELASTIC_USER") or "elastic")
ES_DISABLE_VERIFY_SSL: bool = bool(os.environ.get("ES_DISABLE_VERIFY_SSL"))
def __init__(self, path: str):
self.url: str = f"{self.ES_URL}/{path}"
self.auth: tuple[str, str] = (self.ES_USER, self.ES_PASS)
self.url: str = f"{EnvironmentSettings.ES_URL}/{path}"
self.auth: tuple[str, str] = (
EnvironmentSettings.ES_USER,
EnvironmentSettings.ES_PASS,
)
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def get(
@ -43,7 +41,7 @@ class ElasticWrap:
"timeout": timeout,
}
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
if data:
@ -78,7 +76,7 @@ class ElasticWrap:
}
)
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.post(self.url, **kwargs)
@ -103,7 +101,7 @@ class ElasticWrap:
"auth": self.auth,
}
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.put(self.url, **kwargs)
@ -130,7 +128,7 @@ class ElasticWrap:
if data:
kwargs["json"] = data
if self.ES_DISABLE_VERIFY_SSL:
if EnvironmentSettings.ES_DISABLE_VERIFY_SSL:
kwargs["verify"] = False
response = requests.delete(self.url, **kwargs)

8
tubearchivist/home/src/es/snapshot.py
View File

@ -4,12 +4,12 @@ functionality:
"""
from datetime import datetime
from os import environ
from time import sleep
from zoneinfo import ZoneInfo
from home.src.es.connect import ElasticWrap
from home.src.ta.helper import get_mapping
from home.src.ta.settings import EnvironmentSettings
class ElasticSnapshot:
@ -19,9 +19,7 @@ class ElasticSnapshot:
REPO_SETTINGS = {
"compress": "true",
"chunk_size": "1g",
"location": environ.get(
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
),
"location": EnvironmentSettings.ES_SNAPSHOT_DIR,
}
POLICY = "ta_daily"
@ -256,7 +254,7 @@ class ElasticSnapshot:
expected_format = "%Y-%m-%dT%H:%M:%S.%fZ"
date = datetime.strptime(date_utc, expected_format)
local_datetime = date.replace(tzinfo=ZoneInfo("localtime"))
converted = local_datetime.astimezone(ZoneInfo(environ.get("TZ")))
converted = local_datetime.astimezone(ZoneInfo(EnvironmentSettings.TZ))
converted_str = converted.strftime("%Y-%m-%d %H:%M")
return converted_str

5
tubearchivist/home/src/index/channel.py
View File

@ -14,6 +14,7 @@ from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.generic import YouTubeItem
from home.src.index.playlist import YoutubePlaylist
from home.src.ta.settings import EnvironmentSettings
class YoutubeChannel(YouTubeItem):
@ -134,7 +135,7 @@ class YoutubeChannel(YouTubeItem):
def _info_json_fallback(self):
"""read channel info.json for additional metadata"""
info_json = os.path.join(
self.config["application"]["cache_dir"],
EnvironmentSettings.CACHE_DIR,
"import",
f"{self.youtube_id}.info.json",
)
@ -178,7 +179,7 @@ class YoutubeChannel(YouTubeItem):
def get_folder_path(self):
"""get folder where media files get stored"""
folder_path = os.path.join(
self.app_conf["videos"],
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel_id"],
)
return folder_path

4
tubearchivist/home/src/index/filesystem.py
View File

@ -8,14 +8,14 @@ import os
from home.src.es.connect import ElasticWrap, IndexPaginate
from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo, index_new_video
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
class Scanner:
"""scan index and filesystem"""
VIDEOS: str = AppConfig().config["application"]["videos"]
VIDEOS: str = EnvironmentSettings.MEDIA_DIR
def __init__(self, task=False) -> None:
self.task = task

1
tubearchivist/home/src/index/generic.py
View File

@ -26,7 +26,6 @@ class YouTubeItem:
self.youtube_id = youtube_id
self.es_path = f"{self.index_name}/_doc/{youtube_id}"
self.config = AppConfig().config
self.app_conf = self.config["application"]
self.youtube_meta = False
self.json_data = False

11
tubearchivist/home/src/index/manual.py
View File

@ -16,6 +16,7 @@ from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
from home.src.ta.settings import EnvironmentSettings
from PIL import Image
from yt_dlp.utils import ISO639Utils
@ -28,7 +29,7 @@ class ImportFolderScanner:
"""
CONFIG = AppConfig().config
CACHE_DIR = CONFIG["application"]["cache_dir"]
CACHE_DIR = EnvironmentSettings.CACHE_DIR
IMPORT_DIR = os.path.join(CACHE_DIR, "import")
"""All extensions should be in lowercase until better handling is in place.
@ -433,9 +434,9 @@ class ManualImport:
def _move_to_archive(self, json_data):
"""move identified media file to archive"""
videos = self.config["application"]["videos"]
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
videos = EnvironmentSettings.MEDIA_DIR
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel)
@ -472,7 +473,7 @@ class ManualImport:
os.remove(subtitle_file)
channel_info = os.path.join(
self.config["application"]["cache_dir"],
EnvironmentSettings.CACHE_DIR,
"import",
f"{json_data['channel']['channel_id']}.info.json",
)

15
tubearchivist/home/src/index/reindex.py
View File

@ -19,6 +19,7 @@ from home.src.index.comments import Comments
from home.src.index.playlist import YoutubePlaylist
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisQueue
@ -293,7 +294,7 @@ class Reindex(ReindexBase):
# get new
media_url = os.path.join(
self.config["application"]["videos"], es_meta["media_url"]
EnvironmentSettings.MEDIA_DIR, es_meta["media_url"]
)
video.build_json(media_path=media_url)
if not video.youtube_meta:
@ -311,10 +312,6 @@ class Reindex(ReindexBase):
video.json_data["playlist"] = es_meta.get("playlist")
video.upload_to_es()
if es_meta.get("media_url") != video.json_data["media_url"]:
self._rename_media_file(
es_meta.get("media_url"), video.json_data["media_url"]
)
thumb_handler = ThumbManager(youtube_id)
thumb_handler.delete_video_thumb()
@ -325,14 +322,6 @@ class Reindex(ReindexBase):
return
def _rename_media_file(self, media_url_is, media_url_should):
"""handle title change"""
print(f"[reindex] fix media_url {media_url_is} to {media_url_should}")
videos = self.config["application"]["videos"]
old_path = os.path.join(videos, media_url_is)
new_path = os.path.join(videos, media_url_should)
os.rename(old_path, new_path)
def _reindex_single_channel(self, channel_id):
"""refresh channel data and sync to videos"""
# read current state

9
tubearchivist/home/src/index/subtitle.py
View File

@ -12,6 +12,7 @@ from datetime import datetime
import requests
from home.src.es.connect import ElasticWrap
from home.src.ta.helper import requests_headers
from home.src.ta.settings import EnvironmentSettings
class YoutubeSubtitle:
@ -113,7 +114,7 @@ class YoutubeSubtitle:
def download_subtitles(self, relevant_subtitles):
"""download subtitle files to archive"""
videos_base = self.video.config["application"]["videos"]
videos_base = EnvironmentSettings.MEDIA_DIR
indexed = []
for subtitle in relevant_subtitles:
dest_path = os.path.join(videos_base, subtitle["media_url"])
@ -149,8 +150,8 @@ class YoutubeSubtitle:
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(subtitle_str)
host_uid = self.video.config["application"]["HOST_UID"]
host_gid = self.video.config["application"]["HOST_GID"]
host_uid = EnvironmentSettings.HOST_UID
host_gid = EnvironmentSettings.HOST_GID
if host_uid and host_gid:
os.chown(dest_path, host_uid, host_gid)
@ -162,7 +163,7 @@ class YoutubeSubtitle:
def delete(self, subtitles=False):
"""delete subtitles from index and filesystem"""
youtube_id = self.video.youtube_id
videos_base = self.video.config["application"]["videos"]
videos_base = EnvironmentSettings.MEDIA_DIR
# delete files
if subtitles:
files = [i["media_url"] for i in subtitles]

7
tubearchivist/home/src/index/video.py
View File

@ -18,6 +18,7 @@ from home.src.index.subtitle import YoutubeSubtitle
from home.src.index.video_constants import VideoTypeEnum
from home.src.index.video_streams import MediaStreamExtractor
from home.src.ta.helper import get_duration_sec, get_duration_str, randomizor
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.users import UserConfig
from ryd_client import ryd_client
@ -226,14 +227,14 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
def build_dl_cache_path(self):
"""find video path in dl cache"""
cache_dir = self.app_conf["cache_dir"]
cache_dir = EnvironmentSettings.CACHE_DIR
video_id = self.json_data["youtube_id"]
cache_path = f"{cache_dir}/download/{video_id}.mp4"
if os.path.exists(cache_path):
return cache_path
channel_path = os.path.join(
self.app_conf["videos"],
EnvironmentSettings.MEDIA_DIR,
self.json_data["channel"]["channel_id"],
f"{video_id}.mp4",
)
@ -282,7 +283,7 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
if not self.json_data:
raise FileNotFoundError
video_base = self.app_conf["videos"]
video_base = EnvironmentSettings.MEDIA_DIR
media_url = self.json_data.get("media_url")
file_path = os.path.join(video_base, media_url)
try:

19
tubearchivist/home/src/ta/config.py
View File

@ -5,7 +5,6 @@ Functionality:
"""
import json
import os
import re
from random import randint
from time import sleep
@ -28,7 +27,6 @@ class AppConfig:
if not config:
config = self.get_config_file()
config["application"].update(self.get_config_env())
return config
def get_config_file(self):
@ -36,25 +34,8 @@ class AppConfig:
with open("home/config.json", "r", encoding="utf-8") as f:
config_file = json.load(f)
config_file["application"].update(self.get_config_env())
return config_file
@staticmethod
def get_config_env():
"""read environment application variables.
Connection to ES is managed in ElasticWrap and the
connection to Redis is managed in RedisArchivist."""
application = {
"HOST_UID": int(os.environ.get("HOST_UID", False)),
"HOST_GID": int(os.environ.get("HOST_GID", False)),
"enable_cast": bool(os.environ.get("ENABLE_CAST")),
}
return application
@staticmethod
def get_config_redis():
"""read config json set from redis to overwrite defaults"""

10
tubearchivist/home/src/ta/helper.py
View File

@ -112,13 +112,13 @@ def time_parser(timestamp: str) -> float:
return int(hours) * 60 * 60 + int(minutes) * 60 + float(seconds)
def clear_dl_cache(config: dict) -> int:
def clear_dl_cache(cache_dir: str) -> int:
"""clear leftover files from dl cache"""
print("clear download cache")
cache_dir = os.path.join(config["application"]["cache_dir"], "download")
leftover_files = ignore_filelist(os.listdir(cache_dir))
download_cache_dir = os.path.join(cache_dir, "download")
leftover_files = ignore_filelist(os.listdir(download_cache_dir))
for cached in leftover_files:
to_delete = os.path.join(cache_dir, cached)
to_delete = os.path.join(download_cache_dir, cached)
os.remove(to_delete)
return len(leftover_files)
@ -178,7 +178,7 @@ def get_duration_str(seconds: int) -> str:
for unit_label, unit_seconds in units:
if seconds >= unit_seconds:
unit_count, seconds = divmod(seconds, unit_seconds)
duration_parts.append(f"{unit_count}{unit_label}")
duration_parts.append(f"{unit_count:02}{unit_label}")
return " ".join(duration_parts)

95
tubearchivist/home/src/ta/settings.py Normal file
View File

@ -0,0 +1,95 @@
"""
Functionality:
- read and write application config backed by ES
- encapsulate persistence of application properties
"""
from os import environ
class EnvironmentSettings:
"""
Handle settings for the application that are driven from the environment.
These will not change when the user is using the application.
These settings are only provided only on startup.
"""
HOST_UID: int = int(environ.get("HOST_UID", False))
HOST_GID: int = int(environ.get("HOST_GID", False))
ENABLE_CAST: bool = bool(environ.get("ENABLE_CAST"))
TZ: str = str(environ.get("TZ", "UTC"))
TA_PORT: int = int(environ.get("TA_PORT", False))
TA_UWSGI_PORT: int = int(environ.get("TA_UWSGI_PORT", False))
TA_USERNAME: str = str(environ.get("TA_USERNAME"))
TA_PASSWORD: str = str(environ.get("TA_PASSWORD"))
# Application Paths
MEDIA_DIR: str = str(environ.get("TA_MEDIA_DIR", "/youtube"))
APP_DIR: str = str(environ.get("TA_APP_DIR", "/app"))
CACHE_DIR: str = str(environ.get("TA_CACHE_DIR", "/cache"))
# Redis
REDIS_HOST: str = str(environ.get("REDIS_HOST"))
REDIS_PORT: int = int(environ.get("REDIS_PORT", 6379))
REDIS_NAME_SPACE: str = str(environ.get("REDIS_NAME_SPACE", "ta:"))
# ElasticSearch
ES_URL: str = str(environ.get("ES_URL"))
ES_PASS: str = str(environ.get("ELASTIC_PASSWORD"))
ES_USER: str = str(environ.get("ELASTIC_USER", "elastic"))
ES_SNAPSHOT_DIR: str = str(
environ.get(
"ES_SNAPSHOT_DIR", "/usr/share/elasticsearch/data/snapshot"
)
)
ES_DISABLE_VERIFY_SSL: bool = bool(environ.get("ES_DISABLE_VERIFY_SSL"))
def print_generic(self):
"""print generic env vars"""
print(
f"""
HOST_UID: {self.HOST_UID}
HOST_GID: {self.HOST_GID}
TZ: {self.TZ}
ENABLE_CAST: {self.ENABLE_CAST}
TA_PORT: {self.TA_PORT}
TA_UWSGI_PORT: {self.TA_UWSGI_PORT}
TA_USERNAME: {self.TA_USERNAME}
TA_PASSWORD: *****"""
)
def print_paths(self):
"""debug paths set"""
print(
f"""
MEDIA_DIR: {self.MEDIA_DIR}
APP_DIR: {self.APP_DIR}
CACHE_DIR: {self.CACHE_DIR}"""
)
def print_redis_conf(self):
"""debug redis conf paths"""
print(
f"""
REDIS_HOST: {self.REDIS_HOST}
REDIS_PORT: {self.REDIS_PORT}
REDIS_NAME_SPACE: {self.REDIS_NAME_SPACE}"""
)
def print_es_paths(self):
"""debug es conf"""
print(
f"""
ES_URL: {self.ES_URL}
ES_PASS: *****
ES_USER: {self.ES_USER}
ES_SNAPSHOT_DIR: {self.ES_SNAPSHOT_DIR}
ES_DISABLE_VERIFY_SSL: {self.ES_DISABLE_VERIFY_SSL}"""
)
def print_all(self):
"""print all"""
self.print_generic()
self.print_paths()
self.print_redis_conf()
self.print_es_paths()

11
tubearchivist/home/src/ta/ta_redis.py
View File

@ -6,20 +6,21 @@ functionality:
"""
import json
import os
import redis
from home.src.ta.settings import EnvironmentSettings
class RedisBase:
"""connection base for redis"""
REDIS_HOST: str = str(os.environ.get("REDIS_HOST"))
REDIS_PORT: int = int(os.environ.get("REDIS_PORT") or 6379)
NAME_SPACE: str = "ta:"
NAME_SPACE: str = EnvironmentSettings.REDIS_NAME_SPACE
def __init__(self):
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
self.conn = redis.Redis(
host=EnvironmentSettings.REDIS_HOST,
port=EnvironmentSettings.REDIS_PORT,
)
class RedisArchivist(RedisBase):

7
tubearchivist/home/src/ta/users.py
View File

@ -28,12 +28,7 @@ class UserConfigType(TypedDict, total=False):
class UserConfig:
"""Handle settings for an individual user
Create getters and setters for usage in the application.
Although tedious it helps prevents everything caring about how properties
are persisted. Plus it allows us to save anytime any value is set.
"""
"""Handle settings for an individual user"""
_DEFAULT_USER_SETTINGS = UserConfigType(
colors="dark",

11
tubearchivist/home/tasks.py
View File

@ -24,13 +24,14 @@ from home.src.index.manual import ImportFolderScanner
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.notify import Notifications
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
from home.src.ta.urlparser import Parser
CONFIG = AppConfig().config
REDIS_HOST = os.environ.get("REDIS_HOST")
REDIS_PORT = os.environ.get("REDIS_PORT") or 6379
REDIS_HOST = EnvironmentSettings.REDIS_HOST
REDIS_PORT = EnvironmentSettings.REDIS_PORT
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
app = Celery(
@ -39,9 +40,11 @@ app = Celery(
backend=f"redis://{REDIS_HOST}:{REDIS_PORT}",
result_extended=True,
)
app.config_from_object("django.conf:settings", namespace="ta:")
app.config_from_object(
"django.conf:settings", namespace=EnvironmentSettings.REDIS_NAME_SPACE
)
app.autodiscover_tasks()
app.conf.timezone = os.environ.get("TZ") or "UTC"
app.conf.timezone = EnvironmentSettings.TZ
class BaseTask(Task):

3
tubearchivist/home/templates/home/base.html
View File

@ -1,4 +1,5 @@
{% load static %}
{% load auth_extras %}
<!DOCTYPE html>
<html lang="en">
<head>
@ -57,9 +58,11 @@
<a href="{% url 'playlist' %}">
<div class="nav-item">playlists</div>
</a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<a href="{% url 'downloads' %}">
<div class="nav-item">downloads</div>
</a>
{% endif %}
</div>
<div class="nav-icons">
<a href="{% url 'search' %}">

9
tubearchivist/home/templates/home/base_settings.html
View File

@ -1,14 +1,17 @@
{# Base file for all of the settings pages to ensure a common menu #}
{% extends "home/base.html" %}
{% load static %}
{% load auth_extras %}
{% block content %}
<div class="boxed-content">
<div class="info-box-item child-page-nav">
<a href="{% url 'settings' %}"><h3>Dashboard</h3></a>
<a href="{% url 'settings_user' %}"><h3>User</h3></a>
<a href="{% url 'settings_application' %}"><h3>Application</h3></a>
<a href="{% url 'settings_scheduling' %}"><h3>Scheduling</h3></a>
<a href="{% url 'settings_actions' %}"><h3>Actions</h3></a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<a href="{% url 'settings_application' %}"><h3>Application</h3></a>
<a href="{% url 'settings_scheduling' %}"><h3>Scheduling</h3></a>
<a href="{% url 'settings_actions' %}"><h3>Actions</h3></a>
{% endif %}
</div>
<div id="notifications" data=""></div>
{% block settings_content %}{% endblock %}

3
tubearchivist/home/templates/home/channel.html
View File

@ -2,11 +2,13 @@
{% load static %}
{% load humanize %}
{% block content %}
{% load auth_extras %}
<div class="boxed-content">
<div class="title-split">
<div class="title-bar">
<h1>Channels</h1>
</div>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<div class="title-split-form">
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Channels">
<div class="show-form">
@ -17,6 +19,7 @@
</form>
</div>
</div>
{% endif %}
</div>
<div id="notifications" data="subscription"></div>
<div class="view-controls">

8
tubearchivist/home/templates/home/channel_id.html
View File

@ -2,6 +2,8 @@
{% block content %}
{% load static %}
{% load humanize %}
{% load auth_extras %}
<div class="boxed-content">
<div class="channel-banner">
<a href="/channel/{{ channel_info.channel_id }}/"><img src="/cache/channels/{{ channel_info.channel_id }}_banner.jpg" alt="channel_banner"></a>
@ -19,7 +21,9 @@
{% endif %}
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
{% if has_pending %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
{% endif %}
</div>
<div id="notifications" data="channel reindex"></div>
@ -38,7 +42,9 @@
<p>Subscribers: {{ channel_info.channel_subs|intcomma }}</p>
{% endif %}
{% if channel_info.channel_subscribed %}
{% if request.user|has_group:"admin" or request.user.is_staff %}
<button class="unsubscribe" type="button" data-type="channel" data-subscribe="" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ channel_info.channel_name }}">Unsubscribe</button>
{% endif %}
{% else %}
<button type="button" data-type="channel" data-subscribe="true" data-id="{{ channel_info.channel_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ channel_info.channel_name }}">Subscribe</button>
{% endif %}

125
tubearchivist/home/templates/home/channel_id_about.html
View File

@ -2,6 +2,7 @@
{% block content %}
{% load static %}
{% load humanize %}
{% load auth_extras %}
<div class="boxed-content">
<div class="channel-banner">
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
@ -19,7 +20,9 @@
{% endif %}
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
{% if has_pending %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
{% endif %}
</div>
<div id="notifications" data="channel reindex"></div>
@ -56,19 +59,21 @@
{% elif channel_info.channel_views > 0 %}
<p>Channel views: {{ channel_info.channel_views|intcomma }}</p>
{% endif %}
<div class="button-box">
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
<div class="delete-confirm" id="delete-button">
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
</div>
</div>
{% if reindex %}
<p>Reindex scheduled</p>
{% else %}
<div id="reindex-button" class="button-box">
<button data-id="{{ channel_info.channel_id }}" data-type="channel" onclick="reindex(this)" title="Reindex Channel {{ channel_info.channel_name }}">Reindex</button>
<button data-id="{{ channel_info.channel_id }}" data-type="channel" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ channel_info.channel_name }}">Reindex Videos</button>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<div class="button-box">
<button onclick="deleteConfirm()" id="delete-item">Delete Channel</button>
<div class="delete-confirm" id="delete-button">
<span>Delete {{ channel_info.channel_name }} including all videos? </span><button class="danger-button" onclick="deleteChannel(this)" data-id="{{ channel_info.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
</div>
</div>
{% if reindex %}
<p>Reindex scheduled</p>
{% else %}
<div id="reindex-button" class="button-box">
<button data-id="{{ channel_info.channel_id }}" data-type="channel" onclick="reindex(this)" title="Reindex Channel {{ channel_info.channel_name }}">Reindex</button>
<button data-id="{{ channel_info.channel_id }}" data-type="channel" data-extract-videos="true" onclick="reindex(this)" title="Reindex Videos of {{ channel_info.channel_name }}">Reindex Videos</button>
</div>
{% endif %}
{% endif %}
</div>
</div>
@ -90,53 +95,55 @@
</div>
</div>
{% endif %}
<div id="overwrite-form" class="info-box">
<div class="info-box-item">
<h2>Customize {{ channel_info.channel_name }}</h2>
<form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
{% csrf_token %}
<div class="overwrite-form-item">
<p>Download format: <span class="settings-current">
{% if channel_info.channel_overwrites.download_format %}
{{ channel_info.channel_overwrites.download_format }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.download_format }}<br>
</div>
<div class="overwrite-form-item">
<p>Auto delete watched videos after x days: <span class="settings-current">
{% if channel_info.channel_overwrites.autodelete_days %}
{{ channel_info.channel_overwrites.autodelete_days }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.autodelete_days }}<br>
</div>
<div class="overwrite-form-item">
<p>Index playlists: <span class="settings-current">
{% if channel_info.channel_overwrites.index_playlists %}
{{ channel_info.channel_overwrites.index_playlists }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.index_playlists }}<br>
</div>
<div class="overwrite-form-item">
<p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
{% if channel_info.channel_overwrites.integrate_sponsorblock %}
{{ channel_info.channel_overwrites.integrate_sponsorblock }}
{% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
Disabled
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.integrate_sponsorblock }}<br>
</div>
<button type="submit">Save Channel Overwrites</button>
</form>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<div id="overwrite-form" class="info-box">
<div class="info-box-item">
<h2>Customize {{ channel_info.channel_name }}</h2>
<form class="overwrite-form" action="/channel/{{ channel_info.channel_id }}/about/" method="POST">
{% csrf_token %}
<div class="overwrite-form-item">
<p>Download format: <span class="settings-current">
{% if channel_info.channel_overwrites.download_format %}
{{ channel_info.channel_overwrites.download_format }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.download_format }}<br>
</div>
<div class="overwrite-form-item">
<p>Auto delete watched videos after x days: <span class="settings-current">
{% if channel_info.channel_overwrites.autodelete_days %}
{{ channel_info.channel_overwrites.autodelete_days }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.autodelete_days }}<br>
</div>
<div class="overwrite-form-item">
<p>Index playlists: <span class="settings-current">
{% if channel_info.channel_overwrites.index_playlists %}
{{ channel_info.channel_overwrites.index_playlists }}
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.index_playlists }}<br>
</div>
<div class="overwrite-form-item">
<p>Enable <a href="https://sponsor.ajay.app/" target="_blank">SponsorBlock</a>: <span class="settings-current">
{% if channel_info.channel_overwrites.integrate_sponsorblock %}
{{ channel_info.channel_overwrites.integrate_sponsorblock }}
{% elif channel_info.channel_overwrites.integrate_sponsorblock == False %}
Disabled
{% else %}
False
{% endif %}</span></p>
{{ channel_overwrite_form.integrate_sponsorblock }}<br>
</div>
<button type="submit">Save Channel Overwrites</button>
</form>
</div>
</div>
</div>
{% endif %}
</div>
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
{% endblock content %}

7
tubearchivist/home/templates/home/channel_id_playlist.html
View File

@ -2,6 +2,7 @@
{% block content %}
{% load static %}
{% load humanize %}
{% load auth_extras %}
<div class="boxed-content">
<div class="channel-banner">
<a href="{% url 'channel_id' channel_info.channel_id %}"><img src="{{ channel_info.channel_banner_url }}" alt="channel_banner"></a>
@ -19,7 +20,9 @@
{% endif %}
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
{% if has_pending %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
{% endif %}
</div>
<div id="notifications" data="channel reindex"></div>
@ -53,7 +56,7 @@
<a href="{% url 'channel_id' playlist.playlist_channel_id %}"><h3>{{ playlist.playlist_channel }}</h3></a>
<a href="{% url 'playlist_id' playlist.playlist_id %}"><h2>{{ playlist.playlist_name }}</h2></a>
<p>Last refreshed: {{ playlist.playlist_last_refresh }}</p>
{% if playlist.playlist_subscribed %}
{% if playlist.playlist_subscribed and request.user|has_group:"admin" or request.user.is_staff %}
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist.playlist_name }}">Unsubscribe</button>
{% else %}
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist.playlist_name }}">Subscribe</button>

6
tubearchivist/home/templates/home/playlist.html
View File

@ -1,11 +1,15 @@
{% extends "home/base.html" %}
{% load static %}
{% block content %}
{% load auth_extras %}
<div class="boxed-content">
<div class="title-split">
<div class="title-bar">
<h1>Playlists</h1>
</div>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<div class="title-split-form">
<img id="animate-icon" onclick="showForm()" src="{% static 'img/icon-add.svg' %}" alt="add-icon" title="Subscribe to Playlists">
<div class="show-form">
@ -16,6 +20,8 @@
</form>
</div>
</div>
{% endif %}
</div>
<div id="notifications" data="subscription"></div>
<div class="view-controls">

4
tubearchivist/home/templates/home/playlist_id.html
View File

@ -2,6 +2,8 @@
{% load static %}
{% load humanize %}
{% block content %}
{% load auth_extras %}
<div class="boxed-content">
<div class="title-bar">
<h1>{{ playlist_info.playlist_name }}</h1>
@ -27,7 +29,9 @@
<p>Last refreshed: {{ playlist_info.playlist_last_refresh }}</p>
<p>Playlist:
{% if playlist_info.playlist_subscribed %}
{% if request.user|has_group:"admin" or request.user.is_staff %}
<button class="unsubscribe" type="button" data-type="playlist" data-subscribe="" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Unsubscribe from {{ playlist_info.playlist_name }}">Unsubscribe</button>
{% endif %}
{% else %}
<button type="button" data-type="playlist" data-subscribe="true" data-id="{{ playlist_info.playlist_id }}" onclick="subscribeStatus(this)" title="Subscribe to {{ playlist_info.playlist_name }}">Subscribe</button>
{% endif %}

7
tubearchivist/home/templates/home/video.html
View File

@ -2,6 +2,7 @@
{% block content %}
{% load static %}
{% load humanize %}
{% load auth_extras %}
<div id="player" class="player-wrapper">
<div class="video-main">
<div class="video-modal"><span class="video-modal-text"></span></div>
@ -81,15 +82,19 @@
{% if reindex %}
<p>Reindex scheduled</p>
{% else %}
{% if request.user|has_group:"admin" or request.user.is_staff %}
<div id="reindex-button" class="button-box">
<button data-id="{{ video.youtube_id }}" data-type="video" onclick="reindex(this)" title="Reindex {{ video.title }}">Reindex</button>
</div>
{% endif %}
{% endif %}
<a download="" href="/media/{{ video.media_url }}"><button id="download-item">Download File</button></a>
<a download="" href="{{ video.media_url }}"><button id="download-item">Download File</button></a>
{% if request.user|has_group:"admin" or request.user.is_staff %}
<button onclick="deleteConfirm()" id="delete-item">Delete Video</button>
<div class="delete-confirm" id="delete-button">
<span>Are you sure? </span><button class="danger-button" onclick="deleteVideo(this)" data-id="{{ video.youtube_id }}" data-redirect = "{{ video.channel.channel_id }}">Delete</button> <button onclick="cancelDelete()">Cancel</button>
</div>
{% endif %}
</div>
</div>
<div class="info-box-item">

0
tubearchivist/home/templatetags/__init__.py Normal file
View File

8
tubearchivist/home/templatetags/auth_extras.py Normal file
View File

@ -0,0 +1,8 @@
from django import template
register = template.Library()
@register.filter(name="has_group")
def has_group(user, group_name):
return user.groups.filter(name=group_name).exists()

17
tubearchivist/home/views.py
View File

@ -9,11 +9,14 @@ import urllib.parse
from time import sleep
from api.src.search_processor import SearchProcess, process_aggs
from api.views import check_admin
from django.conf import settings
from django.contrib.auth import login
from django.contrib.auth.decorators import user_passes_test
from django.contrib.auth.forms import AuthenticationForm
from django.http import Http404, JsonResponse
from django.shortcuts import redirect, render
from django.utils.decorators import method_decorator
from django.views import View
from home.src.download.queue import PendingInteract
from home.src.download.yt_dlp_base import CookieHandler
@ -39,6 +42,7 @@ from home.src.index.reindex import ReindexProgress
from home.src.index.video_constants import VideoTypeEnum
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.helper import time_parser
from home.src.ta.settings import EnvironmentSettings
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.users import UserConfig
from home.tasks import index_channel_playlists, subscribe_to
@ -53,7 +57,6 @@ class ArchivistViewConfig(View):
self.view_origin = view_origin
self.user_id = False
self.user_conf: UserConfig = False
self.default_conf = False
self.context = False
def get_all_view_styles(self):
@ -70,11 +73,10 @@ class ArchivistViewConfig(View):
"""build default context for every view"""
self.user_id = user_id
self.user_conf = UserConfig(self.user_id)
self.default_conf = AppConfig().config
self.context = {
"colors": self.user_conf.get_value("colors"),
"cast": self.default_conf["application"]["enable_cast"],
"cast": EnvironmentSettings.ENABLE_CAST,
"sort_by": self.user_conf.get_value("sort_by"),
"sort_order": self.user_conf.get_value("sort_order"),
"view_style": self.user_conf.get_value(
@ -317,6 +319,7 @@ class AboutView(MinView):
return render(request, "home/about.html", context)
@method_decorator(user_passes_test(check_admin), name="dispatch")
class DownloadView(ArchivistResultsView):
"""resolves to /download/
handle the download queue
@ -597,6 +600,7 @@ class ChannelIdAboutView(ChannelIdBaseView):
return render(request, "home/channel_id_about.html", self.context)
@method_decorator(user_passes_test(check_admin), name="dispatch")
@staticmethod
def post(request, channel_id):
"""handle post request"""
@ -681,6 +685,7 @@ class ChannelView(ArchivistResultsView):
"term": {"channel_subscribed": {"value": True}}
}
@method_decorator(user_passes_test(check_admin), name="dispatch")
@staticmethod
def post(request):
"""handle http post requests"""
@ -824,6 +829,7 @@ class PlaylistView(ArchivistResultsView):
}
}
@method_decorator(user_passes_test(check_admin), name="dispatch")
@staticmethod
def post(request):
"""handle post from search form"""
@ -870,7 +876,7 @@ class VideoView(MinView):
"video": video_data,
"playlist_nav": playlist_nav,
"title": video_data.get("title"),
"cast": config_handler.config["application"]["enable_cast"],
"cast": EnvironmentSettings.ENABLE_CAST,
"config": config_handler.config,
"position": time_parser(request.GET.get("t")),
"reindex": reindex.get("state"),
@ -986,6 +992,7 @@ class SettingsUserView(MinView):
return redirect("settings_user", permanent=True)
@method_decorator(user_passes_test(check_admin), name="dispatch")
class SettingsApplicationView(MinView):
"""resolves to /settings/application/
handle the settings sub-page for application configuration,
@ -1075,6 +1082,7 @@ class SettingsApplicationView(MinView):
RedisArchivist().set_message(key, message=message, expire=True)
@method_decorator(user_passes_test(check_admin), name="dispatch")
class SettingsSchedulingView(MinView):
"""resolves to /settings/scheduling/
handle the settings sub-page for scheduling settings,
@ -1108,6 +1116,7 @@ class SettingsSchedulingView(MinView):
return redirect("settings_scheduling", permanent=True)
@method_decorator(user_passes_test(check_admin), name="dispatch")
class SettingsActionsView(MinView):
"""resolves to /settings/actions/
handle the settings actions sub-page

10
tubearchivist/requirements.txt
View File

@ -1,13 +1,13 @@
apprise==1.5.0
apprise==1.6.0
celery==5.3.4
Django==4.2.6
django-auth-ldap==4.6.0
django-cors-headers==4.2.0
django-cors-headers==4.3.0
djangorestframework==3.14.0
Pillow==10.0.1
Pillow==10.1.0
redis==5.0.1
requests==2.31.0
ryd-client==0.0.6
uWSGI==2.0.22
whitenoise==6.5.0
yt-dlp==2023.10.7
whitenoise==6.6.0
yt-dlp==2023.10.13

2
tubearchivist/static/script.js
View File

@ -470,7 +470,7 @@ function createPlayer(button) {
// If cast integration is enabled create cast button
let castButton = '';
if (videoData.config.application.enable_cast) {
if (videoData.config.enable_cast) {
castButton = `<google-cast-launcher id="castbutton"></google-cast-launcher>`;
}

193
tubearchivist/static/stats.js
View File

@ -8,13 +8,18 @@ function primaryStats() {
let apiEndpoint = '/api/stats/primary/';
let responseData = apiRequest(apiEndpoint, 'GET');
let primaryBox = document.getElementById('primaryBox');
clearLoading(primaryBox);
let videoTile = buildVideoTile(responseData);
primaryBox.appendChild(videoTile);
let channelTile = buildChannelTile(responseData);
primaryBox.appendChild(channelTile);
let playlistTile = buildPlaylistTile(responseData);
primaryBox.appendChild(playlistTile);
let downloadTile = buildDownloadTile(responseData);
primaryBox.appendChild(downloadTile);
}
@ -26,51 +31,133 @@ function clearLoading(dashBox) {
function buildTile(titleText) {
let tile = document.createElement('div');
tile.classList.add('info-box-item');
let title = document.createElement('h3');
title.innerText = titleText;
tile.appendChild(title);
return tile;
}
function buildTileContenTable(content, rowsWanted) {
let contentEntries = Object.entries(content);
const nbsp = '\u00A0'; // No-Break Space https://www.compart.com/en/unicode/U+00A0
// Do not add spacing rows when on mobile device
const isMobile = window.matchMedia('(max-width: 600px)');
if (!isMobile.matches) {
if (contentEntries.length < rowsWanted) {
const rowsToAdd = rowsWanted - contentEntries.length;
for (let i = 0; i < rowsToAdd; i++) {
contentEntries.push([nbsp, nbsp]);
}
}
}
const table = document.createElement('table');
table.classList.add('agg-channel-table');
const tableBody = document.createElement('tbody');
for (const [key, value] of contentEntries) {
const row = document.createElement('tr');
const leftCell = document.createElement('td');
leftCell.classList.add('agg-channel-name');
// Do not add ":" when its a spacing entry
const keyText = key === nbsp ? key : `${key}: `;
const leftText = document.createTextNode(keyText);
leftCell.appendChild(leftText);
const rightCell = document.createElement('td');
rightCell.classList.add('agg-channel-right-align');
const rightText = document.createTextNode(value);
rightCell.appendChild(rightText);
row.appendChild(leftCell);
row.appendChild(rightCell);
tableBody.appendChild(row);
}
table.appendChild(tableBody);
return table;
}
function buildVideoTile(responseData) {
let tile = buildTile(`Total Videos: ${responseData.videos.total || 0}`);
let message = document.createElement('p');
message.innerHTML = `
videos: ${responseData.videos.videos || 0}<br>
shorts: ${responseData.videos.shorts || 0}<br>
streams: ${responseData.videos.streams || 0}<br>
`;
tile.appendChild(message);
let tile = buildTile(`Video types: `);
const total = responseData.videos.total || 0;
const videos = responseData.videos.videos || 0;
const shorts = responseData.videos.shorts || 0;
const streams = responseData.videos.streams || 0;
const content = {
Videos: `${videos}/${total}`,
Shorts: `${shorts}/${total}`,
Streams: `${streams}/${total}`,
};
const table = buildTileContenTable(content, 3);
tile.appendChild(table);
return tile;
}
function buildChannelTile(responseData) {
let tile = buildTile(`Total Channels: ${responseData.channels.total || 0}`);
let message = document.createElement('p');
message.innerHTML = `subscribed: ${responseData.channels.sub_true || 0}`;
tile.appendChild(message);
let tile = buildTile(`Channels: `);
const total = responseData.channels.total || 0;
const subscribed = responseData.channels.sub_true || 0;
const content = {
Subscribed: `${subscribed}/${total}`,
};
const table = buildTileContenTable(content, 3);
tile.appendChild(table);
return tile;
}
function buildPlaylistTile(responseData) {
let tile = buildTile(`Total Playlists: ${responseData.playlists.total || 0}`);
let message = document.createElement('p');
message.innerHTML = `subscribed: ${responseData.playlists.sub_true || 0}`;
tile.appendChild(message);
let tile = buildTile(`Playlists:`);
const total = responseData.playlists.total || 0;
const subscribed = responseData.playlists.sub_true || 0;
const content = {
Subscribed: `${subscribed}/${total}`,
};
const table = buildTileContenTable(content, 3);
tile.appendChild(table);
return tile;
}
function buildDownloadTile(responseData) {
let tile = buildTile('Downloads');
let message = document.createElement('p');
message.innerHTML = `
pending: ${responseData.downloads.pending || 0}<br>
ignored: ${responseData.downloads.ignore || 0}<br>
`;
tile.appendChild(message);
const pending = responseData.downloads.pending || 0;
const ignored = responseData.downloads.ignore || 0;
const content = {
Pending: pending,
Ignored: ignored,
};
const table = buildTileContenTable(content, 3);
tile.appendChild(table);
return tile;
}
@ -81,24 +168,48 @@ function watchStats() {
let watchBox = document.getElementById('watchBox');
clearLoading(watchBox);
for (const property in responseData) {
let tile = buildWatchTile(property, responseData[property]);
watchBox.appendChild(tile);
}
const { total, watched, unwatched } = responseData;
let firstCard = buildWatchTile('total', total);
watchBox.appendChild(firstCard);
let secondCard = buildWatchTile('watched', watched);
watchBox.appendChild(secondCard);
let thirdCard = buildWatchTile('unwatched', unwatched);
watchBox.appendChild(thirdCard);
}
function capitalizeFirstLetter(string) {
// source: https://stackoverflow.com/a/1026087
return string.charAt(0).toUpperCase() + string.slice(1);
}
function buildWatchTile(title, watchDetail) {
let tile = buildTile(`Total ${title}`);
let message = document.createElement('p');
message.innerHTML = `
${watchDetail.items} Videos<br>
${watchDetail.duration} Seconds<br>
${watchDetail.duration_str} Playback
`;
if (watchDetail.progress) {
message.innerHTML += `<br>${Number(watchDetail.progress * 100).toFixed(2)}%`;
const items = watchDetail.items || 0;
const duration = watchDetail.duration || 0;
const duration_str = watchDetail.duration_str || 0;
const hasProgess = !!watchDetail.progress;
const progress = Number(watchDetail.progress * 100).toFixed(2);
let titleCapizalized = capitalizeFirstLetter(title);
if (hasProgess) {
titleCapizalized = `${progress}% ` + titleCapizalized;
}
tile.appendChild(message);
let tile = buildTile(titleCapizalized);
const content = {
Videos: items,
Seconds: duration,
Playback: duration_str,
};
const table = buildTileContenTable(content, 3);
tile.appendChild(table);
return tile;
}
@ -123,7 +234,15 @@ function downloadHist() {
function buildDailyStat(dailyStat) {
let tile = buildTile(dailyStat.date);
let message = document.createElement('p');
message.innerText = `new videos: ${dailyStat.count}`;
const isExactlyOne = dailyStat.count === 1;
let text = 'Videos';
if (isExactlyOne) {
text = 'Video';
}
message.innerText = `+${dailyStat.count} ${text}`;
tile.appendChild(message);
return tile;
}