refactor VideoDownloader class, better obs builder

This commit is contained in:
simon 2022-01-26 19:32:30 +07:00
parent 2fc0cbacf5
commit b6f9fb58ad

View File

@ -31,8 +31,10 @@ class VideoDownloader:
""" """
def __init__(self, youtube_id_list=False): def __init__(self, youtube_id_list=False):
self.obs = False
self.youtube_id_list = youtube_id_list self.youtube_id_list = youtube_id_list
self.config = AppConfig().config self.config = AppConfig().config
self._build_obs()
self.channels = set() self.channels = set()
def run_queue(self): def run_queue(self):
@ -49,14 +51,14 @@ class VideoDownloader:
break break
try: try:
self.dl_single_vid(youtube_id) self._dl_single_vid(youtube_id)
except yt_dlp.utils.DownloadError: except yt_dlp.utils.DownloadError:
print("failed to download " + youtube_id) print("failed to download " + youtube_id)
continue continue
vid_dict = index_new_video(youtube_id) vid_dict = index_new_video(youtube_id)
self.channels.add(vid_dict["channel"]["channel_id"]) self.channels.add(vid_dict["channel"]["channel_id"])
self.move_to_archive(vid_dict) self.move_to_archive(vid_dict)
self.delete_from_pending(youtube_id) self._delete_from_pending(youtube_id)
autodelete_days = self.config["downloads"]["autodelete_days"] autodelete_days = self.config["downloads"]["autodelete_days"]
if autodelete_days: if autodelete_days:
@ -91,7 +93,7 @@ class VideoDownloader:
queue.add_list(to_add) queue.add_list(to_add)
@staticmethod @staticmethod
def progress_hook(response): def _progress_hook(response):
"""process the progress_hooks from yt_dlp""" """process the progress_hooks from yt_dlp"""
# title # title
path = os.path.split(response["filename"])[-1][12:] path = os.path.split(response["filename"])[-1][12:]
@ -115,9 +117,15 @@ class VideoDownloader:
} }
RedisArchivist().set_message("message:download", mess_dict) RedisArchivist().set_message("message:download", mess_dict)
def build_obs(self): def _build_obs(self):
"""build obs dictionary for yt-dlp""" """collection to build all obs passed to yt-dlp"""
obs = { self._build_obs_basic()
self._build_obs_user()
self._build_obs_postprocessors()
def _build_obs_basic(self):
"""initial obs"""
self.obs = {
"default_search": "ytsearch", "default_search": "ytsearch",
"merge_output_format": "mp4", "merge_output_format": "mp4",
"restrictfilenames": True, "restrictfilenames": True,
@ -126,7 +134,7 @@ class VideoDownloader:
+ "/download/" + "/download/"
+ self.config["application"]["file_template"] + self.config["application"]["file_template"]
), ),
"progress_hooks": [self.progress_hook], "progress_hooks": [self._progress_hook],
"noprogress": True, "noprogress": True,
"quiet": True, "quiet": True,
"continuedl": True, "continuedl": True,
@ -135,15 +143,22 @@ class VideoDownloader:
"noplaylist": True, "noplaylist": True,
"check_formats": "selected", "check_formats": "selected",
} }
def _build_obs_user(self):
"""build user customized options"""
if self.config["downloads"]["format"]: if self.config["downloads"]["format"]:
obs["format"] = self.config["downloads"]["format"] self.obs["format"] = self.config["downloads"]["format"]
if self.config["downloads"]["limit_speed"]: if self.config["downloads"]["limit_speed"]:
obs["ratelimit"] = self.config["downloads"]["limit_speed"] * 1024 self.obs["ratelimit"] = (
self.config["downloads"]["limit_speed"] * 1024
)
throttle = self.config["downloads"]["throttledratelimit"] throttle = self.config["downloads"]["throttledratelimit"]
if throttle: if throttle:
obs["throttledratelimit"] = throttle * 1024 self.obs["throttledratelimit"] = throttle * 1024
def _build_obs_postprocessors(self):
"""add postprocessor to obs"""
postprocessors = [] postprocessors = []
if self.config["downloads"]["add_metadata"]: if self.config["downloads"]["add_metadata"]:
@ -162,23 +177,20 @@ class VideoDownloader:
"already_have_thumbnail": True, "already_have_thumbnail": True,
} }
) )
obs["writethumbnail"] = True self.obs["writethumbnail"] = True
obs["postprocessors"] = postprocessors self.obs["postprocessors"] = postprocessors
return obs def _dl_single_vid(self, youtube_id):
def dl_single_vid(self, youtube_id):
"""download single video""" """download single video"""
dl_cache = self.config["application"]["cache_dir"] + "/download/" dl_cache = self.config["application"]["cache_dir"] + "/download/"
obs = self.build_obs()
# check if already in cache to continue from there # check if already in cache to continue from there
all_cached = ignore_filelist(os.listdir(dl_cache)) all_cached = ignore_filelist(os.listdir(dl_cache))
for file_name in all_cached: for file_name in all_cached:
if youtube_id in file_name: if youtube_id in file_name:
obs["outtmpl"] = os.path.join(dl_cache, file_name) self.obs["outtmpl"] = os.path.join(dl_cache, file_name)
with yt_dlp.YoutubeDL(obs) as ydl: with yt_dlp.YoutubeDL(self.obs) as ydl:
try: try:
ydl.download([youtube_id]) ydl.download([youtube_id])
except yt_dlp.utils.DownloadError: except yt_dlp.utils.DownloadError:
@ -186,7 +198,7 @@ class VideoDownloader:
sleep(10) sleep(10)
ydl.download([youtube_id]) ydl.download([youtube_id])
if obs["writethumbnail"]: if self.obs["writethumbnail"]:
# webp files don't get cleaned up automatically # webp files don't get cleaned up automatically
all_cached = ignore_filelist(os.listdir(dl_cache)) all_cached = ignore_filelist(os.listdir(dl_cache))
to_clean = [i for i in all_cached if not i.endswith(".mp4")] to_clean = [i for i in all_cached if not i.endswith(".mp4")]
@ -219,7 +231,7 @@ class VideoDownloader:
if host_uid and host_gid: if host_uid and host_gid:
os.chown(new_file_path, host_uid, host_gid) os.chown(new_file_path, host_uid, host_gid)
def delete_from_pending(self, youtube_id): def _delete_from_pending(self, youtube_id):
"""delete downloaded video from pending index if its there""" """delete downloaded video from pending index if its there"""
es_url = self.config["application"]["es_url"] es_url = self.config["application"]["es_url"]
es_auth = self.config["application"]["es_auth"] es_auth = self.config["application"]["es_auth"]
@ -228,7 +240,7 @@ class VideoDownloader:
if not response.ok and not response.status_code == 404: if not response.ok and not response.status_code == 404:
print(response.text) print(response.text)
def add_subscribed_channels(self): def _add_subscribed_channels(self):
"""add all channels subscribed to refresh""" """add all channels subscribed to refresh"""
all_subscribed = PlaylistSubscription().get_playlists() all_subscribed = PlaylistSubscription().get_playlists()
if not all_subscribed: if not all_subscribed:
@ -243,7 +255,7 @@ class VideoDownloader:
def validate_playlists(self): def validate_playlists(self):
"""look for playlist needing to update""" """look for playlist needing to update"""
print("sync playlists") print("sync playlists")
self.add_subscribed_channels() self._add_subscribed_channels()
all_indexed = PendingList().get_all_indexed() all_indexed = PendingList().get_all_indexed()
all_youtube_ids = [i["youtube_id"] for i in all_indexed] all_youtube_ids = [i["youtube_id"] for i in all_indexed]
for id_c, channel_id in enumerate(self.channels): for id_c, channel_id in enumerate(self.channels):