diff --git a/Dockerfile b/Dockerfile index 8f545a1..2651f86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,21 +7,7 @@ FROM python:3.10.4-slim-bullseye AS builder ARG TARGETPLATFORM RUN apt-get update -RUN apt-get install -y --no-install-recommends build-essential gcc curl - -# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64 -RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ - curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \ - | grep browser_download_url \ - | grep ".*master.*linux64.*tar.xz" \ - | cut -d '"' -f 4 \ - | xargs curl -L --output ffmpeg.tar.xz && \ - tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \ - tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \ - rm ffmpeg.tar.xz \ - ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ - apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \ - ; fi +RUN apt-get install -y --no-install-recommends build-essential gcc # install requirements COPY ./tubearchivist/requirements.txt /requirements.txt @@ -37,15 +23,28 @@ ENV PYTHONUNBUFFERED 1 # copy build requirements COPY --from=builder /root/.local /root/.local -COPY --from=builder /usr/bin/ffmpeg /usr/bin/ffmpeg -COPY --from=builder /usr/bin/ffprobe /usr/bin/ffprobe ENV PATH=/root/.local/bin:$PATH # install distro packages needed RUN apt-get clean && apt-get -y update && apt-get -y install --no-install-recommends \ nginx \ atomicparsley \ - curl && rm -rf /var/lib/apt/lists/* + curl \ + xz-utils && rm -rf /var/lib/apt/lists/* + +# get newest patched ffmpeg and ffprobe builds for amd64 fall back to repo ffmpeg for arm64 +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \ + curl -s https://api.github.com/repos/yt-dlp/FFmpeg-Builds/releases/latest \ + | grep browser_download_url \ + | grep ".*master.*linux64.*tar.xz" \ + | cut -d '"' -f 4 \ + | xargs curl -L --output ffmpeg.tar.xz && \ + tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \ + tar -xf ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" && \ + rm ffmpeg.tar.xz \ + ; elif [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ + apt-get -y update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/* \ + ; fi # install debug tools for testing environment RUN if [ "$INSTALL_DEBUG" ] ; then \ diff --git a/tubearchivist/api/README.md b/tubearchivist/api/README.md index dada701..c0889d3 100644 --- a/tubearchivist/api/README.md +++ b/tubearchivist/api/README.md @@ -213,3 +213,21 @@ POST /api/task/ List of valid task names: - **download_pending**: Start the download queue - **rescan_pending**: Rescan your subscriptions + + +## Cookie View +Check your youtube cookie settings +GET /api/cookie/ +```json +{ + "cookie_enabled": true +} +``` + +POST /api/cookie/ +Send empty post request to validate cookie. +```json +{ + "cookie_validated": true +} +``` diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py index e059f75..eedcbe2 100644 --- a/tubearchivist/api/urls.py +++ b/tubearchivist/api/urls.py @@ -4,6 +4,7 @@ from api.views import ( ChannelApiListView, ChannelApiVideoView, ChannelApiView, + CookieView, DownloadApiListView, DownloadApiView, LoginApiView, @@ -87,4 +88,9 @@ urlpatterns = [ TaskApiView.as_view(), name="api-task", ), + path( + "cookie/", + CookieView.as_view(), + name="api-cookie", + ), ] diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py index 88b8a0e..fe07637 100644 --- a/tubearchivist/api/views.py +++ b/tubearchivist/api/views.py @@ -3,6 +3,7 @@ from api.src.search_processor import SearchProcess from api.src.task_processor import TaskHandler from home.src.download.queue import PendingInteract +from home.src.download.yt_cookie import CookieHandler from home.src.es.connect import ElasticWrap from home.src.index.generic import Pagination from home.src.index.video import SponsorBlock @@ -462,3 +463,27 @@ class TaskApiView(ApiBaseView): response = TaskHandler(data).run_task() return Response(response) + + +class CookieView(ApiBaseView): + """resolves to /api/cookie/ + GET: check if cookie is enabled + POST: verify validity of cookie + """ + + @staticmethod + def get(request): + """handle get request""" + # pylint: disable=unused-argument + config = AppConfig().config + cookie_enabled = config["downloads"]["cookie_import"] + + return Response({"cookie_enabled": cookie_enabled}) + + @staticmethod + def post(request): + """handle post request""" + # pylint: disable=unused-argument + validated = CookieHandler().validate() + + return Response({"cookie_validated": validated}) diff --git a/tubearchivist/home/config.json b/tubearchivist/home/config.json index edb6356..9ffe376 100644 --- a/tubearchivist/home/config.json +++ b/tubearchivist/home/config.json @@ -26,6 +26,7 @@ "subtitle": false, "subtitle_source": false, "subtitle_index": false, + "cookie_import": false, "throttledratelimit": false, "integrate_ryd": false, "integrate_sponsorblock": false diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index d898b82..5ec4a19 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -13,8 +13,10 @@ from home.src.download.subscriptions import ( PlaylistSubscription, ) from home.src.download.thumbnails import ThumbManager +from home.src.download.yt_cookie import CookieHandler from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.playlist import YoutubePlaylist +from home.src.ta.config import AppConfig from home.src.ta.helper import DurationConverter from home.src.ta.ta_redis import RedisArchivist @@ -119,12 +121,29 @@ class PendingInteract: class PendingList(PendingIndex): """manage the pending videos list""" + yt_obs = { + "default_search": "ytsearch", + "quiet": True, + "check_formats": "selected", + "noplaylist": True, + "writethumbnail": True, + "simulate": True, + } + def __init__(self, youtube_ids=False): super().__init__() + self.process_config() self.youtube_ids = youtube_ids self.to_skip = False self.missing_videos = False + def process_config(self): + """add user config to yt_obs""" + config = AppConfig().config + if config["downloads"]["cookie_import"]: + cookie_path = CookieHandler().use() + self.yt_obs.update({"cookiefile": cookie_path}) + def parse_url_list(self): """extract youtube ids from list""" self.missing_videos = [] @@ -223,16 +242,8 @@ class PendingList(PendingIndex): def get_youtube_details(self, youtube_id): """get details from youtubedl for single pending video""" - obs = { - "default_search": "ytsearch", - "quiet": True, - "check_formats": "selected", - "noplaylist": True, - "writethumbnail": True, - "simulate": True, - } try: - vid = yt_dlp.YoutubeDL(obs).extract_info(youtube_id) + vid = yt_dlp.YoutubeDL(self.yt_obs).extract_info(youtube_id) except yt_dlp.utils.DownloadError: print("failed to extract info for: " + youtube_id) return False diff --git a/tubearchivist/home/src/download/yt_cookie.py b/tubearchivist/home/src/download/yt_cookie.py new file mode 100644 index 0000000..3b16ca5 --- /dev/null +++ b/tubearchivist/home/src/download/yt_cookie.py @@ -0,0 +1,85 @@ +""" +functionality: +- import yt cookie from filesystem +- make cookie available for yt-dlp +""" + +import os + +import yt_dlp +from home.src.ta.config import AppConfig +from home.src.ta.ta_redis import RedisArchivist + + +class CookieHandler: + """handle youtube cookie for yt-dlp""" + + CONFIG = AppConfig().config + CACHE_PATH = CONFIG["application"]["cache_dir"] + COOKIE_FILE_NAME = "cookies.google.txt" + COOKIE_KEY = "cookie" + COOKIE_PATH = "cookie.txt" + + def import_cookie(self): + """import cookie from file""" + import_path = os.path.join( + self.CACHE_PATH, "import", self.COOKIE_FILE_NAME + ) + with open(import_path, encoding="utf-8") as cookie_file: + cookie = cookie_file.read() + + RedisArchivist().set_message(self.COOKIE_KEY, cookie, expire=False) + + os.remove(import_path) + print("cookie: import successfully") + + def use(self): + """make cookie available in FS""" + cookie = RedisArchivist().get_message(self.COOKIE_KEY) + if isinstance(cookie, dict): + print("no cookie imported") + raise FileNotFoundError + + with open(self.COOKIE_PATH, "w", encoding="utf-8") as cookie_file: + cookie_file.write(cookie) + + print("cookie: made available") + return self.COOKIE_PATH + + def hide(self): + """hide cookie file if not in use""" + try: + os.remove(self.COOKIE_PATH) + except FileNotFoundError: + print("cookie: not available") + return + + print("cookie: hidden") + + def revoke(self): + """revoke cookie""" + self.hide() + RedisArchivist().del_message(self.COOKIE_KEY) + print("cookie: revoked") + + def validate(self): + """validate cookie using the liked videos playlist""" + try: + _ = self.use() + except FileNotFoundError: + return False + + url = "https://www.youtube.com/playlist?list=LL" + yt_obs = { + "quiet": True, + "skip_download": True, + "extract_flat": True, + "cookiefile": self.COOKIE_PATH, + } + try: + response = yt_dlp.YoutubeDL(yt_obs).extract_info(url) + except yt_dlp.utils.DownloadError: + print("failed to validate cookie") + response = False + + return bool(response) diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 31d8ce9..9712f95 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -14,6 +14,7 @@ from time import sleep import yt_dlp from home.src.download.queue import PendingList from home.src.download.subscriptions import PlaylistSubscription +from home.src.download.yt_cookie import CookieHandler from home.src.es.connect import ElasticWrap, IndexPaginate from home.src.index.channel import YoutubeChannel from home.src.index.playlist import YoutubePlaylist @@ -290,6 +291,9 @@ class VideoDownloader: self.obs["ratelimit"] = ( self.config["downloads"]["limit_speed"] * 1024 ) + if self.config["downloads"]["cookie_import"]: + cookie_path = CookieHandler().use() + self.obs["cookiefile"] = cookie_path throttle = self.config["downloads"]["throttledratelimit"] if throttle: diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index 1a25e6a..900df87 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -86,6 +86,12 @@ class ApplicationSettingsForm(forms.Form): ("1", "enable subtitle index"), ] + COOKIE_IMPORT_CHOICES = [ + ("", "-- change cookie settings"), + ("0", "disable cookie"), + ("1", "enable cookie"), + ] + subscriptions_channel_size = forms.IntegerField(required=False) downloads_limit_count = forms.IntegerField(required=False) downloads_limit_speed = forms.IntegerField(required=False) @@ -106,6 +112,9 @@ class ApplicationSettingsForm(forms.Form): downloads_subtitle_index = forms.ChoiceField( widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False ) + downloads_cookie_import = forms.ChoiceField( + widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False + ) downloads_integrate_ryd = forms.ChoiceField( widget=forms.Select, choices=RYD_CHOICES, required=False ) diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index 709dde9..c4dcac5 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -6,6 +6,7 @@ functionality: import math import yt_dlp +from home.src.download.yt_cookie import CookieHandler from home.src.es.connect import ElasticWrap from home.src.ta.config import AppConfig from home.src.ta.ta_redis import RedisArchivist @@ -37,6 +38,9 @@ class YouTubeItem: """read user conf""" self.config = AppConfig().config self.app_conf = self.config["application"] + if self.config["downloads"]["cookie_import"]: + cookie_path = CookieHandler().use() + self.yt_obs.update({"cookiefile": cookie_path}) def get_from_youtube(self): """use yt-dlp to get meta data from youtube""" diff --git a/tubearchivist/home/src/ta/config.py b/tubearchivist/home/src/ta/config.py index 4b98c4a..020e637 100644 --- a/tubearchivist/home/src/ta/config.py +++ b/tubearchivist/home/src/ta/config.py @@ -83,6 +83,7 @@ class AppConfig: def update_config(self, form_post): """update config values from settings form""" + updated = [] for key, value in form_post.items(): if not value and not isinstance(value, int): continue @@ -96,8 +97,10 @@ class AppConfig: config_dict, config_value = key.split("_", maxsplit=1) self.config[config_dict][config_value] = to_write + updated.append((config_value, to_write)) RedisArchivist().set_message("config", self.config, expire=False) + return updated @staticmethod def set_user_config(form_post, user_id): diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 0ff1516..522e488 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -114,6 +114,14 @@ {{ app_form.downloads_subtitle_index }} +
+

Cookie

+
+

Import YouTube cookie: {{ config.downloads.cookie_import }}

+ Place your cookie file named cookies.google.txt in /cache/import before enabling.
+ {{ app_form.downloads_cookie_import }} +
+

Integrations

diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 92359b8..22d219a 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -14,6 +14,7 @@ from django.contrib.auth.forms import AuthenticationForm from django.http import JsonResponse from django.shortcuts import redirect, render from django.views import View +from home.src.download.yt_cookie import CookieHandler from home.src.es.index_setup import get_available_backups from home.src.frontend.api_calls import PostData from home.src.frontend.forms import ( @@ -791,8 +792,7 @@ class SettingsView(View): token = Token.objects.get_or_create(user=request.user)[0] return token - @staticmethod - def post(request): + def post(self, request): """handle form post to update settings""" user_form = UserSettingsForm(request.POST) if user_form.is_valid(): @@ -805,7 +805,8 @@ class SettingsView(View): app_form_post = app_form.cleaned_data if app_form_post: print(app_form_post) - AppConfig().update_config(app_form_post) + updated = AppConfig().update_config(app_form_post) + self.post_process_updated(updated) scheduler_form = SchedulerSettingsForm(request.POST) if scheduler_form.is_valid(): @@ -817,6 +818,19 @@ class SettingsView(View): sleep(1) return redirect("settings", permanent=True) + @staticmethod + def post_process_updated(updated): + """apply changes for config""" + if not updated: + return + + for config_value, updated_value in updated: + if config_value == "cookie_import": + if updated_value: + CookieHandler().import_cookie() + else: + CookieHandler().revoke() + def progress(request): # pylint: disable=unused-argument