mirror of
https://github.com/tubearchivist/tubearchivist.git
synced 2025-02-19 06:20:13 +00:00
context manager for yt-dlp extract, stricter worker recycle
This commit is contained in:
parent
c7fc2666fa
commit
2ec81c7ac7
@ -82,23 +82,24 @@ class YtWrap:
|
||||
|
||||
def extract(self, url):
|
||||
"""make extract request"""
|
||||
try:
|
||||
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
|
||||
except cookiejar.LoadError as err:
|
||||
print(f"cookie file is invalid: {err}")
|
||||
return False
|
||||
except yt_dlp.utils.ExtractorError as err:
|
||||
print(f"{url}: failed to extract with message: {err}, continue...")
|
||||
return False
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
if "This channel does not have a" in str(err):
|
||||
with yt_dlp.YoutubeDL(self.obs) as ydl:
|
||||
try:
|
||||
response = ydl.extract_info(url)
|
||||
except cookiejar.LoadError as err:
|
||||
print(f"cookie file is invalid: {err}")
|
||||
return False
|
||||
except yt_dlp.utils.ExtractorError as err:
|
||||
print(f"{url}: failed to extract: {err}, continue...")
|
||||
return False
|
||||
except yt_dlp.utils.DownloadError as err:
|
||||
if "This channel does not have a" in str(err):
|
||||
return False
|
||||
|
||||
print(f"{url}: failed to get info from youtube with message {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
print(f"{url}: failed to get info from youtube: {err}")
|
||||
if "Temporary failure in name resolution" in str(err):
|
||||
raise ConnectionError("lost the internet, abort!") from err
|
||||
|
||||
return False
|
||||
return False
|
||||
|
||||
self._validate_cookie()
|
||||
|
||||
|
@ -20,7 +20,11 @@ python manage.py ta_startup
|
||||
|
||||
# start all tasks
|
||||
nginx &
|
||||
celery -A task.celery worker --loglevel=INFO --max-tasks-per-child 10 &
|
||||
celery -A task.celery worker \
|
||||
--loglevel=INFO \
|
||||
--concurrency 4 \
|
||||
--max-tasks-per-child 5 \
|
||||
--max-memory-per-child 150000 &
|
||||
celery -A task beat --loglevel=INFO \
|
||||
--scheduler django_celery_beat.schedulers:DatabaseScheduler &
|
||||
python backend_start.py
|
||||
|
Loading…
Reference in New Issue
Block a user