mirror of
https://github.com/tubearchivist/tubearchivist-frontend.git
synced 2024-11-04 19:30:13 +00:00
refactor VideoDownloader class, better obs builder
This commit is contained in:
parent
2fc0cbacf5
commit
b6f9fb58ad
@ -31,8 +31,10 @@ class VideoDownloader:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, youtube_id_list=False):
|
def __init__(self, youtube_id_list=False):
|
||||||
|
self.obs = False
|
||||||
self.youtube_id_list = youtube_id_list
|
self.youtube_id_list = youtube_id_list
|
||||||
self.config = AppConfig().config
|
self.config = AppConfig().config
|
||||||
|
self._build_obs()
|
||||||
self.channels = set()
|
self.channels = set()
|
||||||
|
|
||||||
def run_queue(self):
|
def run_queue(self):
|
||||||
@ -49,14 +51,14 @@ class VideoDownloader:
|
|||||||
break
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.dl_single_vid(youtube_id)
|
self._dl_single_vid(youtube_id)
|
||||||
except yt_dlp.utils.DownloadError:
|
except yt_dlp.utils.DownloadError:
|
||||||
print("failed to download " + youtube_id)
|
print("failed to download " + youtube_id)
|
||||||
continue
|
continue
|
||||||
vid_dict = index_new_video(youtube_id)
|
vid_dict = index_new_video(youtube_id)
|
||||||
self.channels.add(vid_dict["channel"]["channel_id"])
|
self.channels.add(vid_dict["channel"]["channel_id"])
|
||||||
self.move_to_archive(vid_dict)
|
self.move_to_archive(vid_dict)
|
||||||
self.delete_from_pending(youtube_id)
|
self._delete_from_pending(youtube_id)
|
||||||
|
|
||||||
autodelete_days = self.config["downloads"]["autodelete_days"]
|
autodelete_days = self.config["downloads"]["autodelete_days"]
|
||||||
if autodelete_days:
|
if autodelete_days:
|
||||||
@ -91,7 +93,7 @@ class VideoDownloader:
|
|||||||
queue.add_list(to_add)
|
queue.add_list(to_add)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def progress_hook(response):
|
def _progress_hook(response):
|
||||||
"""process the progress_hooks from yt_dlp"""
|
"""process the progress_hooks from yt_dlp"""
|
||||||
# title
|
# title
|
||||||
path = os.path.split(response["filename"])[-1][12:]
|
path = os.path.split(response["filename"])[-1][12:]
|
||||||
@ -115,9 +117,15 @@ class VideoDownloader:
|
|||||||
}
|
}
|
||||||
RedisArchivist().set_message("message:download", mess_dict)
|
RedisArchivist().set_message("message:download", mess_dict)
|
||||||
|
|
||||||
def build_obs(self):
|
def _build_obs(self):
|
||||||
"""build obs dictionary for yt-dlp"""
|
"""collection to build all obs passed to yt-dlp"""
|
||||||
obs = {
|
self._build_obs_basic()
|
||||||
|
self._build_obs_user()
|
||||||
|
self._build_obs_postprocessors()
|
||||||
|
|
||||||
|
def _build_obs_basic(self):
|
||||||
|
"""initial obs"""
|
||||||
|
self.obs = {
|
||||||
"default_search": "ytsearch",
|
"default_search": "ytsearch",
|
||||||
"merge_output_format": "mp4",
|
"merge_output_format": "mp4",
|
||||||
"restrictfilenames": True,
|
"restrictfilenames": True,
|
||||||
@ -126,7 +134,7 @@ class VideoDownloader:
|
|||||||
+ "/download/"
|
+ "/download/"
|
||||||
+ self.config["application"]["file_template"]
|
+ self.config["application"]["file_template"]
|
||||||
),
|
),
|
||||||
"progress_hooks": [self.progress_hook],
|
"progress_hooks": [self._progress_hook],
|
||||||
"noprogress": True,
|
"noprogress": True,
|
||||||
"quiet": True,
|
"quiet": True,
|
||||||
"continuedl": True,
|
"continuedl": True,
|
||||||
@ -135,15 +143,22 @@ class VideoDownloader:
|
|||||||
"noplaylist": True,
|
"noplaylist": True,
|
||||||
"check_formats": "selected",
|
"check_formats": "selected",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _build_obs_user(self):
|
||||||
|
"""build user customized options"""
|
||||||
if self.config["downloads"]["format"]:
|
if self.config["downloads"]["format"]:
|
||||||
obs["format"] = self.config["downloads"]["format"]
|
self.obs["format"] = self.config["downloads"]["format"]
|
||||||
if self.config["downloads"]["limit_speed"]:
|
if self.config["downloads"]["limit_speed"]:
|
||||||
obs["ratelimit"] = self.config["downloads"]["limit_speed"] * 1024
|
self.obs["ratelimit"] = (
|
||||||
|
self.config["downloads"]["limit_speed"] * 1024
|
||||||
|
)
|
||||||
|
|
||||||
throttle = self.config["downloads"]["throttledratelimit"]
|
throttle = self.config["downloads"]["throttledratelimit"]
|
||||||
if throttle:
|
if throttle:
|
||||||
obs["throttledratelimit"] = throttle * 1024
|
self.obs["throttledratelimit"] = throttle * 1024
|
||||||
|
|
||||||
|
def _build_obs_postprocessors(self):
|
||||||
|
"""add postprocessor to obs"""
|
||||||
postprocessors = []
|
postprocessors = []
|
||||||
|
|
||||||
if self.config["downloads"]["add_metadata"]:
|
if self.config["downloads"]["add_metadata"]:
|
||||||
@ -162,23 +177,20 @@ class VideoDownloader:
|
|||||||
"already_have_thumbnail": True,
|
"already_have_thumbnail": True,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
obs["writethumbnail"] = True
|
self.obs["writethumbnail"] = True
|
||||||
|
|
||||||
obs["postprocessors"] = postprocessors
|
self.obs["postprocessors"] = postprocessors
|
||||||
|
|
||||||
return obs
|
def _dl_single_vid(self, youtube_id):
|
||||||
|
|
||||||
def dl_single_vid(self, youtube_id):
|
|
||||||
"""download single video"""
|
"""download single video"""
|
||||||
dl_cache = self.config["application"]["cache_dir"] + "/download/"
|
dl_cache = self.config["application"]["cache_dir"] + "/download/"
|
||||||
obs = self.build_obs()
|
|
||||||
|
|
||||||
# check if already in cache to continue from there
|
# check if already in cache to continue from there
|
||||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||||
for file_name in all_cached:
|
for file_name in all_cached:
|
||||||
if youtube_id in file_name:
|
if youtube_id in file_name:
|
||||||
obs["outtmpl"] = os.path.join(dl_cache, file_name)
|
self.obs["outtmpl"] = os.path.join(dl_cache, file_name)
|
||||||
with yt_dlp.YoutubeDL(obs) as ydl:
|
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
||||||
try:
|
try:
|
||||||
ydl.download([youtube_id])
|
ydl.download([youtube_id])
|
||||||
except yt_dlp.utils.DownloadError:
|
except yt_dlp.utils.DownloadError:
|
||||||
@ -186,7 +198,7 @@ class VideoDownloader:
|
|||||||
sleep(10)
|
sleep(10)
|
||||||
ydl.download([youtube_id])
|
ydl.download([youtube_id])
|
||||||
|
|
||||||
if obs["writethumbnail"]:
|
if self.obs["writethumbnail"]:
|
||||||
# webp files don't get cleaned up automatically
|
# webp files don't get cleaned up automatically
|
||||||
all_cached = ignore_filelist(os.listdir(dl_cache))
|
all_cached = ignore_filelist(os.listdir(dl_cache))
|
||||||
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
|
to_clean = [i for i in all_cached if not i.endswith(".mp4")]
|
||||||
@ -219,7 +231,7 @@ class VideoDownloader:
|
|||||||
if host_uid and host_gid:
|
if host_uid and host_gid:
|
||||||
os.chown(new_file_path, host_uid, host_gid)
|
os.chown(new_file_path, host_uid, host_gid)
|
||||||
|
|
||||||
def delete_from_pending(self, youtube_id):
|
def _delete_from_pending(self, youtube_id):
|
||||||
"""delete downloaded video from pending index if its there"""
|
"""delete downloaded video from pending index if its there"""
|
||||||
es_url = self.config["application"]["es_url"]
|
es_url = self.config["application"]["es_url"]
|
||||||
es_auth = self.config["application"]["es_auth"]
|
es_auth = self.config["application"]["es_auth"]
|
||||||
@ -228,7 +240,7 @@ class VideoDownloader:
|
|||||||
if not response.ok and not response.status_code == 404:
|
if not response.ok and not response.status_code == 404:
|
||||||
print(response.text)
|
print(response.text)
|
||||||
|
|
||||||
def add_subscribed_channels(self):
|
def _add_subscribed_channels(self):
|
||||||
"""add all channels subscribed to refresh"""
|
"""add all channels subscribed to refresh"""
|
||||||
all_subscribed = PlaylistSubscription().get_playlists()
|
all_subscribed = PlaylistSubscription().get_playlists()
|
||||||
if not all_subscribed:
|
if not all_subscribed:
|
||||||
@ -243,7 +255,7 @@ class VideoDownloader:
|
|||||||
def validate_playlists(self):
|
def validate_playlists(self):
|
||||||
"""look for playlist needing to update"""
|
"""look for playlist needing to update"""
|
||||||
print("sync playlists")
|
print("sync playlists")
|
||||||
self.add_subscribed_channels()
|
self._add_subscribed_channels()
|
||||||
all_indexed = PendingList().get_all_indexed()
|
all_indexed = PendingList().get_all_indexed()
|
||||||
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
|
all_youtube_ids = [i["youtube_id"] for i in all_indexed]
|
||||||
for id_c, channel_id in enumerate(self.channels):
|
for id_c, channel_id in enumerate(self.channels):
|
||||||
|
Loading…
Reference in New Issue
Block a user