diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index fe07637..d1165c2 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,7 +3,7 @@ from api.src.search_processor import SearchProcess from api.src.task_processor import TaskHandler from home.src.download.queue import PendingInteract -from home.src.download.yt_cookie import CookieHandler +from home.src.download.yt_dlp_base import CookieHandler from home.src.es.connect import ElasticWrap from home.src.index.generic import Pagination from home.src.index.video import SponsorBlock @@ -484,6 +484,7 @@ class CookieView(ApiBaseView): def post(request): """handle post request""" # pylint: disable=unused-argument - validated = CookieHandler().validate() + config = AppConfig().config + validated = CookieHandler(config).validate() return Response({"cookie_validated": validated}) diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 00f18b9..36e3d1e 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -12,7 +12,6 @@ from home.src.download.subscriptions import ( PlaylistSubscription, ) from home.src.download.thumbnails import ThumbManager -from home.src.download.yt_cookie import CookieHandler from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.playlist import YoutubePlaylist @@ -133,28 +132,11 @@ class PendingList(PendingIndex): def __init__(self, youtube_ids=False): super().__init__() - self.process_config() + self.config = AppConfig().config self.youtube_ids = youtube_ids self.to_skip = False self.missing_videos = False - def process_config(self): - """add user config to yt_obs""" - config = AppConfig().config - if config["downloads"]["cookie_import"]: - cookie_path = CookieHandler().use() - self.yt_obs.update({"cookiefile": cookie_path}) - - def close_config(self): - """remove config after task finished""" - config = AppConfig().config - if config["downloads"]["cookie_import"]: - CookieHandler().hide() - try: - del self.yt_obs["cookiefile"] - except KeyError: - pass - def parse_url_list(self): """extract youtube ids from list""" self.missing_videos = [] @@ -235,8 +217,6 @@ class PendingList(PendingIndex): query_str = "\n".join(bulk_list) _, _ = ElasticWrap("_bulk").post(query_str, ndjson=True) - self.close_config() - def _notify_add(self, idx): """send notification for adding videos to download queue""" progress = f"{idx + 1}/{len(self.missing_videos)}" @@ -256,7 +236,7 @@ class PendingList(PendingIndex): def get_youtube_details(self, youtube_id): """get details from youtubedl for single pending video""" - vid = YtWrap(self.yt_obs).extract(youtube_id) + vid = YtWrap(self.yt_obs, self.config).extract(youtube_id) if not vid: return False diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index 99d88f3..973c76e 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -17,10 +17,7 @@ class ChannelSubscription: """manage the list of channels subscribed""" def __init__(self): - config = AppConfig().config - self.es_url = config["application"]["es_url"] - self.es_auth = config["application"]["es_auth"] - self.channel_size = config["subscriptions"]["channel_size"] + self.config = AppConfig().config @staticmethod def get_channels(subscribed_only=True): @@ -44,9 +41,9 @@ class ChannelSubscription: "extract_flat": True, } if limit: - obs["playlistend"] = self.channel_size + obs["playlistend"] = self.config["subscriptions"]["channel_size"] - channel = YtWrap(obs).extract(channel_id) + channel = YtWrap(obs, self.config).extract(channel_id) if not channel: return False diff --git a/tubearchivist/home/src/download/yt_cookie.py b/tubearchivist/home/src/download/yt_cookie.py deleted file mode 100644 index 3a7244e..0000000 --- a/tubearchivist/home/src/download/yt_cookie.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -functionality: -- import yt cookie from filesystem -- make cookie available for yt-dlp -""" - -import os - -import yt_dlp -from home.src.ta.config import AppConfig -from home.src.ta.ta_redis import RedisArchivist - - -class CookieHandler: - """handle youtube cookie for yt-dlp""" - - CONFIG = AppConfig().config - CACHE_PATH = CONFIG["application"]["cache_dir"] - COOKIE_FILE_NAME = "cookies.google.txt" - COOKIE_KEY = "cookie" - COOKIE_PATH = "cookie.txt" - - def import_cookie(self): - """import cookie from file""" - import_path = os.path.join( - self.CACHE_PATH, "import", self.COOKIE_FILE_NAME - ) - with open(import_path, encoding="utf-8") as cookie_file: - cookie = cookie_file.read() - - RedisArchivist().set_message(self.COOKIE_KEY, cookie, expire=False) - - os.remove(import_path) - print("cookie: import successfully") - - def use(self): - """make cookie available in FS""" - cookie = RedisArchivist().get_message(self.COOKIE_KEY) - if isinstance(cookie, dict): - print("no cookie imported") - raise FileNotFoundError - - if os.path.exists(self.COOKIE_PATH): - return self.COOKIE_PATH - - with open(self.COOKIE_PATH, "w", encoding="utf-8") as cookie_file: - cookie_file.write(cookie) - - print("cookie: made available") - return self.COOKIE_PATH - - def hide(self): - """hide cookie file if not in use""" - try: - os.remove(self.COOKIE_PATH) - except FileNotFoundError: - print("cookie: not available") - return - - print("cookie: hidden") - - def revoke(self): - """revoke cookie""" - self.hide() - RedisArchivist().del_message(self.COOKIE_KEY) - print("cookie: revoked") - - def validate(self): - """validate cookie using the liked videos playlist""" - try: - _ = self.use() - except FileNotFoundError: - return False - - url = "https://www.youtube.com/playlist?list=LL" - yt_obs = { - "quiet": True, - "skip_download": True, - "extract_flat": True, - "cookiefile": self.COOKIE_PATH, - } - try: - response = yt_dlp.YoutubeDL(yt_obs).extract_info(url) - except yt_dlp.utils.DownloadError: - print("failed to validate cookie") - response = False - - return bool(response) diff --git a/tubearchivist/home/src/download/yt_dlp_base.py b/tubearchivist/home/src/download/yt_dlp_base.py index 77438fb..b32d536 100644 --- a/tubearchivist/home/src/download/yt_dlp_base.py +++ b/tubearchivist/home/src/download/yt_dlp_base.py @@ -4,7 +4,11 @@ functionality: - handle yt-dlp errors """ +import os +from io import StringIO + import yt_dlp +from home.src.ta.ta_redis import RedisArchivist class YtWrap: @@ -17,14 +21,22 @@ class YtWrap: "socket_timeout": 2, } - def __init__(self, obs_request): + def __init__(self, obs_request, config=False): self.obs_request = obs_request + self.config = config self.build_obs() def build_obs(self): """build yt-dlp obs""" self.obs = self.OBS_BASE.copy() self.obs.update(self.obs_request) + self.add_cookie() + + def add_cookie(self): + """add cookie if enabled""" + if self.config["downloads"]["cookie_import"]: + cookie_io = CookieHandler(self.config).get() + self.obs["cookiefile"] = cookie_io def download(self, url): """make download request""" @@ -46,3 +58,44 @@ class YtWrap: response = False return response + + +class CookieHandler: + """handle youtube cookie for yt-dlp""" + + def __init__(self, config): + self.cookie_io = False + self.config = config + + def get(self): + """get cookie io stream""" + cookie = RedisArchivist().get_message("cookie") + self.cookie_io = StringIO(cookie) + return self.cookie_io + + def import_cookie(self): + """import cookie from file""" + cache_path = self.config["application"]["cache_dir"] + import_path = os.path.join(cache_path, "import", "cookies.google.txt") + with open(import_path, encoding="utf-8") as cookie_file: + cookie = cookie_file.read() + + RedisArchivist().set_message("cookie", cookie, expire=False) + + os.remove(import_path) + print("cookie: import successful") + + @staticmethod + def revoke(): + """revoke cookie""" + RedisArchivist().del_message("cookie") + print("cookie: revoked") + + def validate(self): + """validate cookie using the liked videos playlist""" + obs_request = { + "skip_download": True, + "extract_flat": True, + } + response = YtWrap(obs_request, self.config).extract("LL") + return bool(response) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 13a40d6..fd4c429 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -12,7 +12,6 @@ from datetime import datetime from home.src.download.queue import PendingList from home.src.download.subscriptions import PlaylistSubscription -from home.src.download.yt_cookie import CookieHandler from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel @@ -40,7 +39,6 @@ class DownloadPostProcess: self.auto_delete_all() self.auto_delete_overwrites() self.validate_playlists() - self.pending.close_config() def auto_delete_all(self): """handle auto delete""" @@ -291,9 +289,6 @@ class VideoDownloader: self.obs["ratelimit"] = ( self.config["downloads"]["limit_speed"] * 1024 ) - if self.config["downloads"]["cookie_import"]: - cookie_path = CookieHandler().use() - self.obs["cookiefile"] = cookie_path throttle = self.config["downloads"]["throttledratelimit"] if throttle: @@ -346,7 +341,7 @@ class VideoDownloader: if youtube_id in file_name: obs["outtmpl"] = os.path.join(dl_cache, file_name) - success = YtWrap(obs).download(youtube_id) + success = YtWrap(obs, self.config).download(youtube_id) if self.obs["writethumbnail"]: # webp files don't get cleaned up automatically diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index ec8c0af..4761cd0 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -315,7 +315,7 @@ class YoutubeChannel(YouTubeItem): + "/playlists?view=1&sort=dd&shelf_id=0" ) obs = {"skip_download": True, "extract_flat": True} - playlists = YtWrap(obs).extract(url) + playlists = YtWrap(obs, self.config).extract(url) all_entries = [(i["id"], i["title"]) for i in playlists["entries"]] self.all_playlists = all_entries diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index 93a1bc4..c6f1caa 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -12,7 +12,6 @@ import shutil import subprocess from home.src.download.queue import PendingList -from home.src.download.yt_cookie import CookieHandler from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.connect import ElasticWrap from home.src.index.reindex import Reindex @@ -309,12 +308,6 @@ def scan_filesystem(): def reindex_old_documents(): """daily refresh of old documents""" handler = Reindex() - if handler.config["downloads"]["cookie_import"]: - CookieHandler().use() - try: - handler.check_outdated() - handler.reindex() - RedisArchivist().set_message("last_reindex", handler.now, expire=False) - finally: - if handler.config["downloads"]["cookie_import"]: - CookieHandler().hide() + handler.check_outdated() + handler.reindex() + RedisArchivist().set_message("last_reindex", handler.now, expire=False) diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 06a2806..15e1264 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -5,7 +5,6 @@ functionality: import math -from home.src.download.yt_cookie import CookieHandler from home.src.download.yt_dlp_base import YtWrap from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig @@ -25,24 +24,16 @@ class YouTubeItem: def __init__(self, youtube_id): self.youtube_id = youtube_id - self.config = False - self.app_conf = False - self.youtube_meta = False - self.json_data = False - self._get_conf() - - def _get_conf(self): - """read user conf""" self.config = AppConfig().config self.app_conf = self.config["application"] - if self.config["downloads"]["cookie_import"]: - cookie_path = CookieHandler().use() - self.yt_obs.update({"cookiefile": cookie_path}) + self.youtube_meta = False + self.json_data = False def get_from_youtube(self): """use yt-dlp to get meta data from youtube""" print(f"{self.youtube_id}: get metadata from youtube") - response = YtWrap(self.yt_obs).extract(self.yt_base + self.youtube_id) + url = self.yt_base + self.youtube_id + response = YtWrap(self.yt_obs, self.config).extract(url) self.youtube_meta = response diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 22d219a..bd5a2ea 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -14,7 +14,7 @@ from django.contrib.auth.forms import AuthenticationForm from django.http import JsonResponse from django.shortcuts import redirect, render from django.views import View -from home.src.download.yt_cookie import CookieHandler +from home.src.download.yt_dlp_base import CookieHandler from home.src.es.index_setup import get_available_backups from home.src.frontend.api_calls import PostData from home.src.frontend.forms import ( @@ -795,18 +795,19 @@ class SettingsView(View): def post(self, request): """handle form post to update settings""" user_form = UserSettingsForm(request.POST) + config_handler = AppConfig() if user_form.is_valid(): user_form_post = user_form.cleaned_data if any(user_form_post.values()): - AppConfig().set_user_config(user_form_post, request.user.id) + config_handler.set_user_config(user_form_post, request.user.id) app_form = ApplicationSettingsForm(request.POST) if app_form.is_valid(): app_form_post = app_form.cleaned_data if app_form_post: print(app_form_post) - updated = AppConfig().update_config(app_form_post) - self.post_process_updated(updated) + updated = config_handler.update_config(app_form_post) + self.post_process_updated(updated, config_handler.config) scheduler_form = SchedulerSettingsForm(request.POST) if scheduler_form.is_valid(): @@ -819,7 +820,7 @@ class SettingsView(View): return redirect("settings", permanent=True) @staticmethod - def post_process_updated(updated): + def post_process_updated(updated, config): """apply changes for config""" if not updated: return @@ -827,9 +828,9 @@ class SettingsView(View): for config_value, updated_value in updated: if config_value == "cookie_import": if updated_value: - CookieHandler().import_cookie() + CookieHandler(config).import_cookie() else: - CookieHandler().revoke() + CookieHandler(config).revoke() def progress(request):