diff --git a/docs/Settings.md b/docs/Settings.md index 50d1d4c..b048fbe 100644 --- a/docs/Settings.md +++ b/docs/Settings.md @@ -32,6 +32,27 @@ Additional settings passed to yt-dlp. - **Source Settings**: User created subtitles are provided from the uploader and are usually the video script. Auto generated is from YouTube, quality varies, particularly for auto translated tracks. - **Index Settings**: Enabling subtitle indexing will add the lines to Elasticsearch and will make subtitles searchable. This will increase the index size and is not recommended on low-end hardware. +## Cookie +Importing your YouTube Cookie into Tube Archivist allows yt-dlp to bypass age restrictions, gives access to private videos and your *watch later* or *liked videos*. + +### Security concerns +Cookies are used to store your session and contain your access token to your google account. Treat that file with utmost care as you would any other password or credential. *Tube Archivist* stores your cookie in Redis and will export the file container internally if it's needed. + +### Export your cookie +- Install **Cookies.txt** addon for [chrome](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid) or [firefox](https://addons.mozilla.org/firefox/addon/cookies-txt). +- Visit YouTube and login with whichever YouTube account you wish to use to generate the cookies. +- Click on the extension icon in the toolbar - it will drop down showing the active cookies for YT. +- Click Export to export the cookies, filename is by default *cookies.google.txt*. + +### Import your cookie +Place the file *cookies.google.txt* into the */cache/import* folder of Tube Archivist and enable the cookie import. Once you click on *Update Application Configurations* to save your changes, your cookie will get imported and stored internally. + +### Use your cookie +Once imported, additionally to the advantages above, your [Watch Later](https://www.youtube.com/playlist?list=WL) and [Liked Videos](https://www.youtube.com/playlist?list=LL) become a regular playlist you can download and subscribe to as any other [playlist](Playlists). + +### Limitation +There is only one cookie per Tube Archivist instance, this will be shared between all users. + ## Integrations All third party integrations of TubeArchivist will **always** be *opt in*. - **API**: Your access token for the Tube Archivist API. diff --git a/tubearchivist/home/src/download/queue.py b/tubearchivist/home/src/download/queue.py index 5ec4a19..6be7f82 100644 --- a/tubearchivist/home/src/download/queue.py +++ b/tubearchivist/home/src/download/queue.py @@ -103,7 +103,7 @@ class PendingInteract: def delete_item(self): """delete single item from pending""" path = f"ta_download/_doc/{self.video_id}" - _, _ = ElasticWrap(path).delete() + _, _ = ElasticWrap(path).delete(refresh=True) def delete_by_status(self): """delete all matching item by status""" @@ -128,6 +128,7 @@ class PendingList(PendingIndex): "noplaylist": True, "writethumbnail": True, "simulate": True, + "socket_timeout": 3, } def __init__(self, youtube_ids=False): diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index a460af9..75aede2 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -168,7 +168,7 @@ class PlaylistSubscription: def channel_validate(channel_id): """make sure channel of playlist is there""" channel = YoutubeChannel(channel_id) - channel.build_json() + channel.build_json(upload=True) @staticmethod def change_subscribe(playlist_id, subscribe_status): diff --git a/tubearchivist/home/src/download/thumbnails.py b/tubearchivist/home/src/download/thumbnails.py index d25f4d1..35fc100 100644 --- a/tubearchivist/home/src/download/thumbnails.py +++ b/tubearchivist/home/src/download/thumbnails.py @@ -296,6 +296,7 @@ class ThumbManager: def get_thumb_list(self): """get list of mediafiles and matching thumbnails""" pending = queue.PendingList() + pending.get_download() pending.get_indexed() video_list = [] diff --git a/tubearchivist/home/src/download/yt_dlp_handler.py b/tubearchivist/home/src/download/yt_dlp_handler.py index 9712f95..fd91432 100644 --- a/tubearchivist/home/src/download/yt_dlp_handler.py +++ b/tubearchivist/home/src/download/yt_dlp_handler.py @@ -41,6 +41,7 @@ class DownloadPostProcess: self.auto_delete_all() self.auto_delete_overwrites() self.validate_playlists() + self.clear_cookie() def auto_delete_all(self): """handle auto delete""" @@ -140,6 +141,11 @@ class DownloadPostProcess: else: RedisArchivist().set_message("message:download", mess_dict) + def clear_cookie(self): + """hide cookie file""" + if self.download.config["downloads"]["cookie_import"]: + CookieHandler().hide() + class VideoDownloader: """ @@ -281,6 +287,7 @@ class VideoDownloader: "writethumbnail": False, "noplaylist": True, "check_formats": "selected", + "socket_timeout": 3, } def _build_obs_user(self): diff --git a/tubearchivist/home/src/es/connect.py b/tubearchivist/home/src/es/connect.py index f976943..e61ae14 100644 --- a/tubearchivist/home/src/es/connect.py +++ b/tubearchivist/home/src/es/connect.py @@ -75,8 +75,10 @@ class ElasticWrap: return response.json(), response.status_code - def delete(self, data=False): + def delete(self, data=False, refresh=False): """delete document from es""" + if refresh: + self.url = f"{self.url}/?refresh=true" if data: response = requests.delete(self.url, json=data, auth=self.auth) else: diff --git a/tubearchivist/home/src/frontend/api_calls.py b/tubearchivist/home/src/frontend/api_calls.py index 26b1730..8b0d9b3 100644 --- a/tubearchivist/home/src/frontend/api_calls.py +++ b/tubearchivist/home/src/frontend/api_calls.py @@ -19,7 +19,6 @@ from home.src.ta.ta_redis import RedisArchivist, RedisQueue from home.tasks import ( download_pending, download_single, - extrac_dl, index_channel_playlists, kill_dl, re_sync_thumbs, @@ -111,7 +110,7 @@ class PostData: def _ignore(self): """ignore from download queue""" video_id = self.exec_val - print(f"ignore video {video_id}") + print(f"{video_id}: ignore video from download queue") PendingInteract(video_id=video_id, status="ignore").update_status() # also clear from redis queue RedisQueue().clear_item(video_id) @@ -123,7 +122,7 @@ class PostData: print("download pending") running = download_pending.delay() task_id = running.id - print("set task id: " + task_id) + print(f"{task_id}: set task id") RedisArchivist().set_message("dl_queue_id", task_id, expire=False) return {"success": True} @@ -146,7 +145,7 @@ class PostData: def _unsubscribe(self): """unsubscribe from channels or playlists""" id_unsub = self.exec_val - print("unsubscribe from " + id_unsub) + print(f"{id_unsub}: unsubscribe") to_unsub_list = UrlListParser(id_unsub).process_list() for to_unsub in to_unsub_list: unsub_type = to_unsub["type"] @@ -167,7 +166,7 @@ class PostData: def _subscribe(self): """subscribe to channel or playlist, called from js buttons""" id_sub = self.exec_val - print("subscribe to " + id_sub) + print(f"{id_sub}: subscribe") subscribe_to.delay(id_sub) return {"success": True} @@ -203,7 +202,7 @@ class PostData: def _dlnow(self): """start downloading single vid now""" youtube_id = self.exec_val - print("downloading: " + youtube_id) + print(f"{youtube_id}: downloading now") running = download_single.delay(youtube_id=youtube_id) task_id = running.id print("set task id: " + task_id) @@ -222,17 +221,15 @@ class PostData: def _forget_ignore(self): """delete from ta_download index""" video_id = self.exec_val - print(f"forgetting from download index: {video_id}") + print(f"{video_id}: forget from download") PendingInteract(video_id=video_id).delete_item() return {"success": True} def _add_single(self): """add single youtube_id to download queue""" video_id = self.exec_val - print(f"add vid to dl queue: {video_id}") - PendingInteract(video_id=video_id).delete_item() - video_ids = UrlListParser(video_id).process_list() - extrac_dl.delay(video_ids) + print(f"{video_id}: add single vid to download queue") + PendingInteract(video_id=video_id, status="pending").update_status() return {"success": True} def _delete_queue(self): diff --git a/tubearchivist/home/src/index/filesystem.py b/tubearchivist/home/src/index/filesystem.py index c6f1caa..93a1bc4 100644 --- a/tubearchivist/home/src/index/filesystem.py +++ b/tubearchivist/home/src/index/filesystem.py @@ -12,6 +12,7 @@ import shutil import subprocess from home.src.download.queue import PendingList +from home.src.download.yt_cookie import CookieHandler from home.src.download.yt_dlp_handler import VideoDownloader from home.src.es.connect import ElasticWrap from home.src.index.reindex import Reindex @@ -308,6 +309,12 @@ def scan_filesystem(): def reindex_old_documents(): """daily refresh of old documents""" handler = Reindex() - handler.check_outdated() - handler.reindex() - RedisArchivist().set_message("last_reindex", handler.now, expire=False) + if handler.config["downloads"]["cookie_import"]: + CookieHandler().use() + try: + handler.check_outdated() + handler.reindex() + RedisArchivist().set_message("last_reindex", handler.now, expire=False) + finally: + if handler.config["downloads"]["cookie_import"]: + CookieHandler().hide() diff --git a/tubearchivist/home/src/index/generic.py b/tubearchivist/home/src/index/generic.py index c4dcac5..0940258 100644 --- a/tubearchivist/home/src/index/generic.py +++ b/tubearchivist/home/src/index/generic.py @@ -23,6 +23,7 @@ class YouTubeItem: "default_search": "ytsearch", "skip_download": True, "check_formats": "selected", + "socket_timeout": 3, "noplaylist": True, } @@ -85,7 +86,7 @@ class YouTubeItem: def del_in_es(self): """delete item from elastic search""" print(f"{self.youtube_id}: delete from es") - _, _ = ElasticWrap(self.es_path).delete() + _, _ = ElasticWrap(self.es_path).delete(refresh=True) class Pagination: diff --git a/tubearchivist/home/src/ta/helper.py b/tubearchivist/home/src/ta/helper.py index c572ccc..5731115 100644 --- a/tubearchivist/home/src/ta/helper.py +++ b/tubearchivist/home/src/ta/helper.py @@ -173,7 +173,7 @@ class UrlListParser: id_type = "video" elif str_len == 24: id_type = "channel" - elif str_len in [34, 18]: + elif str_len in [34, 18] or id_str in ["LL", "WL"]: id_type = "playlist" else: # unable to parse diff --git a/tubearchivist/static/script.js b/tubearchivist/static/script.js index 4287100..69caac7 100644 --- a/tubearchivist/static/script.js +++ b/tubearchivist/static/script.js @@ -350,7 +350,10 @@ function createPlayer(button) { `; } } + } else { + sponsorBlock = null; } + var videoProgress = getVideoProgress(videoId).position; var videoName = videoData.data.title;