context manager for yt-dlp extract, stricter worker recycle

This commit is contained in:
Simon 2025-02-02 15:45:40 +07:00
parent c7fc2666fa
commit 2ec81c7ac7
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 20 additions and 15 deletions

View File

@ -82,23 +82,24 @@ class YtWrap:
def extract(self, url):
"""make extract request"""
try:
response = yt_dlp.YoutubeDL(self.obs).extract_info(url)
except cookiejar.LoadError as err:
print(f"cookie file is invalid: {err}")
return False
except yt_dlp.utils.ExtractorError as err:
print(f"{url}: failed to extract with message: {err}, continue...")
return False
except yt_dlp.utils.DownloadError as err:
if "This channel does not have a" in str(err):
with yt_dlp.YoutubeDL(self.obs) as ydl:
try:
response = ydl.extract_info(url)
except cookiejar.LoadError as err:
print(f"cookie file is invalid: {err}")
return False
except yt_dlp.utils.ExtractorError as err:
print(f"{url}: failed to extract: {err}, continue...")
return False
except yt_dlp.utils.DownloadError as err:
if "This channel does not have a" in str(err):
return False
print(f"{url}: failed to get info from youtube with message {err}")
if "Temporary failure in name resolution" in str(err):
raise ConnectionError("lost the internet, abort!") from err
print(f"{url}: failed to get info from youtube: {err}")
if "Temporary failure in name resolution" in str(err):
raise ConnectionError("lost the internet, abort!") from err
return False
return False
self._validate_cookie()

View File

@ -20,7 +20,11 @@ python manage.py ta_startup
# start all tasks
nginx &
celery -A task.celery worker --loglevel=INFO --max-tasks-per-child 10 &
celery -A task.celery worker \
--loglevel=INFO \
--concurrency 4 \
--max-tasks-per-child 5 \
--max-memory-per-child 150000 &
celery -A task beat --loglevel=INFO \
--scheduler django_celery_beat.schedulers:DatabaseScheduler &
python backend_start.py