mirror of
https://github.com/tubearchivist/tubearchivist-frontend.git
synced 2024-11-04 19:30:13 +00:00
commit
5b37bd059c
@ -26,13 +26,13 @@ class ApiBaseView(APIView):
|
|||||||
self.response = {"data": False}
|
self.response = {"data": False}
|
||||||
self.status_code = False
|
self.status_code = False
|
||||||
self.context = False
|
self.context = False
|
||||||
|
self.default_conf = AppConfig().config
|
||||||
|
|
||||||
def config_builder(self):
|
def config_builder(self):
|
||||||
"""build confic context"""
|
"""build confic context"""
|
||||||
default_conf = AppConfig().config
|
|
||||||
self.context = {
|
self.context = {
|
||||||
"es_url": default_conf["application"]["es_url"],
|
"es_url": self.default_conf["application"]["es_url"],
|
||||||
"es_auth": default_conf["application"]["es_auth"],
|
"es_auth": self.default_conf["application"]["es_auth"],
|
||||||
}
|
}
|
||||||
|
|
||||||
def get_document(self, document_id):
|
def get_document(self, document_id):
|
||||||
@ -48,6 +48,19 @@ class ApiBaseView(APIView):
|
|||||||
self.response["data"] = False
|
self.response["data"] = False
|
||||||
self.status_code = response.status_code
|
self.status_code = response.status_code
|
||||||
|
|
||||||
|
def process_keys(self):
|
||||||
|
"""process keys for frontend"""
|
||||||
|
all_keys = self.response["data"].keys()
|
||||||
|
if "media_url" in all_keys:
|
||||||
|
media_url = self.response["data"]["media_url"]
|
||||||
|
self.response["data"]["media_url"] = f"/media/{media_url}"
|
||||||
|
if "vid_thumb_url" in all_keys:
|
||||||
|
youtube_id = self.response["data"]["youtube_id"]
|
||||||
|
vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
|
||||||
|
cache_dir = self.default_conf["application"]["cache_dir"]
|
||||||
|
new_thumb = f"{cache_dir}/{vid_thumb_url}"
|
||||||
|
self.response["data"]["vid_thumb_url"] = new_thumb
|
||||||
|
|
||||||
def get_paginate(self):
|
def get_paginate(self):
|
||||||
"""add pagination detail to response"""
|
"""add pagination detail to response"""
|
||||||
self.response["paginate"] = False
|
self.response["paginate"] = False
|
||||||
@ -75,6 +88,7 @@ class VideoApiView(ApiBaseView):
|
|||||||
"""get request"""
|
"""get request"""
|
||||||
self.config_builder()
|
self.config_builder()
|
||||||
self.get_document(video_id)
|
self.get_document(video_id)
|
||||||
|
self.process_keys()
|
||||||
return Response(self.response, status=self.status_code)
|
return Response(self.response, status=self.status_code)
|
||||||
|
|
||||||
|
|
||||||
|
@ -23,6 +23,8 @@
|
|||||||
"format": false,
|
"format": false,
|
||||||
"add_metadata": false,
|
"add_metadata": false,
|
||||||
"add_thumbnail": false,
|
"add_thumbnail": false,
|
||||||
|
"subtitle": false,
|
||||||
|
"subtitle_source": false,
|
||||||
"throttledratelimit": false,
|
"throttledratelimit": false,
|
||||||
"integrate_ryd": false
|
"integrate_ryd": false
|
||||||
},
|
},
|
||||||
|
@ -212,6 +212,9 @@ class VideoDownloader:
|
|||||||
host_uid = self.config["application"]["HOST_UID"]
|
host_uid = self.config["application"]["HOST_UID"]
|
||||||
host_gid = self.config["application"]["HOST_GID"]
|
host_gid = self.config["application"]["HOST_GID"]
|
||||||
channel_name = clean_string(vid_dict["channel"]["channel_name"])
|
channel_name = clean_string(vid_dict["channel"]["channel_name"])
|
||||||
|
if len(channel_name) <= 3:
|
||||||
|
# fall back to channel id
|
||||||
|
channel_name = vid_dict["channel"]["channel_id"]
|
||||||
# make archive folder with correct permissions
|
# make archive folder with correct permissions
|
||||||
new_folder = os.path.join(videos, channel_name)
|
new_folder = os.path.join(videos, channel_name)
|
||||||
if not os.path.exists(new_folder):
|
if not os.path.exists(new_folder):
|
||||||
|
@ -25,6 +25,10 @@
|
|||||||
"type": "keyword",
|
"type": "keyword",
|
||||||
"index": false
|
"index": false
|
||||||
},
|
},
|
||||||
|
"channel_tvart_url": {
|
||||||
|
"type": "keyword",
|
||||||
|
"index": false
|
||||||
|
},
|
||||||
"channel_thumb_url": {
|
"channel_thumb_url": {
|
||||||
"type": "keyword",
|
"type": "keyword",
|
||||||
"index": false
|
"index": false
|
||||||
@ -84,6 +88,10 @@
|
|||||||
"type": "keyword",
|
"type": "keyword",
|
||||||
"index": false
|
"index": false
|
||||||
},
|
},
|
||||||
|
"channel_tvart_url": {
|
||||||
|
"type": "keyword",
|
||||||
|
"index": false
|
||||||
|
},
|
||||||
"channel_thumb_url": {
|
"channel_thumb_url": {
|
||||||
"type": "keyword",
|
"type": "keyword",
|
||||||
"index": false
|
"index": false
|
||||||
|
@ -68,6 +68,12 @@ class ApplicationSettingsForm(forms.Form):
|
|||||||
("1", "enable Cast"),
|
("1", "enable Cast"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
SUBTITLE_SOURCE_CHOICES = [
|
||||||
|
("", "-- change subtitle source settings"),
|
||||||
|
("auto", "also download auto generated"),
|
||||||
|
("user", "only download uploader"),
|
||||||
|
]
|
||||||
|
|
||||||
subscriptions_channel_size = forms.IntegerField(required=False)
|
subscriptions_channel_size = forms.IntegerField(required=False)
|
||||||
downloads_limit_count = forms.IntegerField(required=False)
|
downloads_limit_count = forms.IntegerField(required=False)
|
||||||
downloads_limit_speed = forms.IntegerField(required=False)
|
downloads_limit_speed = forms.IntegerField(required=False)
|
||||||
@ -81,6 +87,10 @@ class ApplicationSettingsForm(forms.Form):
|
|||||||
downloads_add_thumbnail = forms.ChoiceField(
|
downloads_add_thumbnail = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False
|
widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False
|
||||||
)
|
)
|
||||||
|
downloads_subtitle = forms.CharField(required=False)
|
||||||
|
downloads_subtitle_source = forms.ChoiceField(
|
||||||
|
widget=forms.Select, choices=SUBTITLE_SOURCE_CHOICES, required=False
|
||||||
|
)
|
||||||
downloads_integrate_ryd = forms.ChoiceField(
|
downloads_integrate_ryd = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=RYD_CHOICES, required=False
|
widget=forms.Select, choices=RYD_CHOICES, required=False
|
||||||
)
|
)
|
||||||
|
@ -198,6 +198,9 @@ class YoutubeChannel(YouTubeItem):
|
|||||||
"""get folder where media files get stored"""
|
"""get folder where media files get stored"""
|
||||||
channel_name = self.json_data["channel_name"]
|
channel_name = self.json_data["channel_name"]
|
||||||
folder_name = clean_string(channel_name)
|
folder_name = clean_string(channel_name)
|
||||||
|
if len(folder_name) <= 3:
|
||||||
|
# fall back to channel id
|
||||||
|
folder_name = self.json_data["channel_id"]
|
||||||
folder_path = os.path.join(self.app_conf["videos"], folder_name)
|
folder_path = os.path.join(self.app_conf["videos"], folder_name)
|
||||||
return folder_path
|
return folder_path
|
||||||
|
|
||||||
|
@ -122,6 +122,7 @@ class Pagination:
|
|||||||
"page_from": page_from,
|
"page_from": page_from,
|
||||||
"prev_pages": prev_pages,
|
"prev_pages": prev_pages,
|
||||||
"current_page": page_get,
|
"current_page": page_get,
|
||||||
|
"max_hits": False,
|
||||||
}
|
}
|
||||||
if self.search_get:
|
if self.search_get:
|
||||||
pagination.update({"search_get": self.search_get})
|
pagination.update({"search_get": self.search_get})
|
||||||
@ -131,6 +132,11 @@ class Pagination:
|
|||||||
"""validate pagination with total_hits after making api call"""
|
"""validate pagination with total_hits after making api call"""
|
||||||
page_get = self.page_get
|
page_get = self.page_get
|
||||||
max_pages = math.ceil(total_hits / self.page_size)
|
max_pages = math.ceil(total_hits / self.page_size)
|
||||||
|
if total_hits > 10000:
|
||||||
|
# es returns maximal 10000 results
|
||||||
|
self.pagination["max_hits"] = True
|
||||||
|
max_pages = max_pages - 1
|
||||||
|
|
||||||
if page_get < max_pages and max_pages > 1:
|
if page_get < max_pages and max_pages > 1:
|
||||||
self.pagination["last_page"] = max_pages
|
self.pagination["last_page"] = max_pages
|
||||||
else:
|
else:
|
||||||
|
@ -14,7 +14,107 @@ from home.src.ta.helper import DurationConverter, clean_string
|
|||||||
from ryd_client import ryd_client
|
from ryd_client import ryd_client
|
||||||
|
|
||||||
|
|
||||||
class YoutubeVideo(YouTubeItem):
|
class YoutubeSubtitle:
|
||||||
|
"""handle video subtitle functionality"""
|
||||||
|
|
||||||
|
def __init__(self, config, youtube_meta, media_url, youtube_id):
|
||||||
|
self.config = config
|
||||||
|
self.youtube_meta = youtube_meta
|
||||||
|
self.media_url = media_url
|
||||||
|
self.youtube_id = youtube_id
|
||||||
|
self.languages = False
|
||||||
|
|
||||||
|
def sub_conf_parse(self):
|
||||||
|
"""add additional conf values to self"""
|
||||||
|
languages_raw = self.config["downloads"]["subtitle"]
|
||||||
|
self.languages = [i.strip() for i in languages_raw.split(",")]
|
||||||
|
|
||||||
|
def get_subtitles(self):
|
||||||
|
"""check what to do"""
|
||||||
|
self.sub_conf_parse()
|
||||||
|
if not self.languages:
|
||||||
|
# no subtitles
|
||||||
|
return False
|
||||||
|
|
||||||
|
relevant_subtitles = self.get_user_subtitles()
|
||||||
|
if relevant_subtitles:
|
||||||
|
return relevant_subtitles
|
||||||
|
|
||||||
|
if self.config["downloads"]["subtitle_source"] == "auto":
|
||||||
|
relevant_auto = self.get_auto_caption()
|
||||||
|
return relevant_auto
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_auto_caption(self):
|
||||||
|
"""get auto_caption subtitles"""
|
||||||
|
print(f"{self.youtube_id}: get auto generated subtitles")
|
||||||
|
all_subtitles = self.youtube_meta.get("automatic_captions")
|
||||||
|
|
||||||
|
if not all_subtitles:
|
||||||
|
return False
|
||||||
|
|
||||||
|
relevant_subtitles = []
|
||||||
|
|
||||||
|
for lang in self.languages:
|
||||||
|
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
|
||||||
|
all_formats = all_subtitles.get(lang)
|
||||||
|
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
|
||||||
|
subtitle.update(
|
||||||
|
{"lang": lang, "source": "auto", "media_url": media_url}
|
||||||
|
)
|
||||||
|
relevant_subtitles.append(subtitle)
|
||||||
|
break
|
||||||
|
|
||||||
|
return relevant_subtitles
|
||||||
|
|
||||||
|
def _normalize_lang(self):
|
||||||
|
"""normalize country specific language keys"""
|
||||||
|
all_subtitles = self.youtube_meta.get("subtitles")
|
||||||
|
all_keys = list(all_subtitles.keys())
|
||||||
|
for key in all_keys:
|
||||||
|
lang = key.split("-")[0]
|
||||||
|
old = all_subtitles.pop(key)
|
||||||
|
all_subtitles[lang] = old
|
||||||
|
|
||||||
|
return all_subtitles
|
||||||
|
|
||||||
|
def get_user_subtitles(self):
|
||||||
|
"""get subtitles uploaded from channel owner"""
|
||||||
|
print(f"{self.youtube_id}: get user uploaded subtitles")
|
||||||
|
all_subtitles = self._normalize_lang()
|
||||||
|
if not all_subtitles:
|
||||||
|
return False
|
||||||
|
|
||||||
|
relevant_subtitles = []
|
||||||
|
|
||||||
|
for lang in self.languages:
|
||||||
|
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
|
||||||
|
all_formats = all_subtitles.get(lang)
|
||||||
|
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
|
||||||
|
subtitle.update(
|
||||||
|
{"lang": lang, "source": "user", "media_url": media_url}
|
||||||
|
)
|
||||||
|
relevant_subtitles.append(subtitle)
|
||||||
|
break
|
||||||
|
|
||||||
|
return relevant_subtitles
|
||||||
|
|
||||||
|
def download_subtitles(self, relevant_subtitles):
|
||||||
|
"""download subtitle files to archive"""
|
||||||
|
for subtitle in relevant_subtitles:
|
||||||
|
dest_path = os.path.join(
|
||||||
|
self.config["application"]["videos"], subtitle["media_url"]
|
||||||
|
)
|
||||||
|
response = requests.get(subtitle["url"])
|
||||||
|
if response.ok:
|
||||||
|
with open(dest_path, "w", encoding="utf-8") as subfile:
|
||||||
|
subfile.write(response.text)
|
||||||
|
else:
|
||||||
|
print(f"{self.youtube_id}: failed to download subtitle")
|
||||||
|
|
||||||
|
|
||||||
|
class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
"""represents a single youtube video"""
|
"""represents a single youtube video"""
|
||||||
|
|
||||||
es_path = False
|
es_path = False
|
||||||
@ -37,6 +137,7 @@ class YoutubeVideo(YouTubeItem):
|
|||||||
self._add_stats()
|
self._add_stats()
|
||||||
self.add_file_path()
|
self.add_file_path()
|
||||||
self.add_player()
|
self.add_player()
|
||||||
|
self._check_subtitles()
|
||||||
if self.config["downloads"]["integrate_ryd"]:
|
if self.config["downloads"]["integrate_ryd"]:
|
||||||
self._get_ryd_stats()
|
self._get_ryd_stats()
|
||||||
|
|
||||||
@ -96,7 +197,7 @@ class YoutubeVideo(YouTubeItem):
|
|||||||
vid_path = os.path.join(cache_path, file_cached)
|
vid_path = os.path.join(cache_path, file_cached)
|
||||||
return vid_path
|
return vid_path
|
||||||
|
|
||||||
return False
|
raise FileNotFoundError
|
||||||
|
|
||||||
def add_player(self):
|
def add_player(self):
|
||||||
"""add player information for new videos"""
|
"""add player information for new videos"""
|
||||||
@ -125,6 +226,10 @@ class YoutubeVideo(YouTubeItem):
|
|||||||
"""build media_url for where file will be located"""
|
"""build media_url for where file will be located"""
|
||||||
channel_name = self.json_data["channel"]["channel_name"]
|
channel_name = self.json_data["channel"]["channel_name"]
|
||||||
clean_channel_name = clean_string(channel_name)
|
clean_channel_name = clean_string(channel_name)
|
||||||
|
if len(clean_channel_name) <= 3:
|
||||||
|
# fall back to channel id
|
||||||
|
clean_channel_name = self.json_data["channel"]["channel_id"]
|
||||||
|
|
||||||
timestamp = self.json_data["published"].replace("-", "")
|
timestamp = self.json_data["published"].replace("-", "")
|
||||||
youtube_id = self.json_data["youtube_id"]
|
youtube_id = self.json_data["youtube_id"]
|
||||||
title = self.json_data["title"]
|
title = self.json_data["title"]
|
||||||
@ -163,6 +268,19 @@ class YoutubeVideo(YouTubeItem):
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _check_subtitles(self):
|
||||||
|
"""optionally add subtitles"""
|
||||||
|
handler = YoutubeSubtitle(
|
||||||
|
self.config,
|
||||||
|
self.youtube_meta,
|
||||||
|
media_url=self.json_data["media_url"],
|
||||||
|
youtube_id=self.youtube_id,
|
||||||
|
)
|
||||||
|
subtitles = handler.get_subtitles()
|
||||||
|
if subtitles:
|
||||||
|
self.json_data["subtitles"] = subtitles
|
||||||
|
handler.download_subtitles(relevant_subtitles=subtitles)
|
||||||
|
|
||||||
|
|
||||||
def index_new_video(youtube_id):
|
def index_new_video(youtube_id):
|
||||||
"""combined classes to create new video in index"""
|
"""combined classes to create new video in index"""
|
||||||
|
@ -109,9 +109,21 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
{% if pagination.last_page > 0 %}
|
{% if pagination.last_page > 0 %}
|
||||||
{% if pagination.search_get %}
|
{% if pagination.search_get %}
|
||||||
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">Last ({{ pagination.last_page }})</a>
|
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
|
||||||
|
{% if pagination.max_hits %}
|
||||||
|
Max ({{ pagination.last_page }})
|
||||||
{% else %}
|
{% else %}
|
||||||
<a class="pagination-item" href="?page={{ pagination.last_page }}">Last ({{ pagination.last_page }})</a>
|
Last ({{ pagination.last_page }})
|
||||||
|
{% endif %}
|
||||||
|
</a>
|
||||||
|
{% else %}
|
||||||
|
<a class="pagination-item" href="?page={{ pagination.last_page }}">
|
||||||
|
{% if pagination.max_hits %}
|
||||||
|
Max ({{ pagination.last_page }})
|
||||||
|
{% else %}
|
||||||
|
Last ({{ pagination.last_page }})
|
||||||
|
{% endif %}
|
||||||
|
</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
@ -94,6 +94,17 @@
|
|||||||
<i>Embed thumbnail into the mediafile.</i><br>
|
<i>Embed thumbnail into the mediafile.</i><br>
|
||||||
{{ app_form.downloads_add_thumbnail }}
|
{{ app_form.downloads_add_thumbnail }}
|
||||||
</div>
|
</div>
|
||||||
|
<div class="settings-item">
|
||||||
|
<p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
|
||||||
|
<i>Choose which subtitles to download, add comma separated two letter language ISO code,<br>
|
||||||
|
e.g. <span class="settings-current">en, de</span></i><br>
|
||||||
|
{{ app_form.downloads_subtitle }}</p>
|
||||||
|
</div>
|
||||||
|
<div class="settings-item">
|
||||||
|
<p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
|
||||||
|
<i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
|
||||||
|
{{ app_form.downloads_subtitle_source }}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="settings-group">
|
<div class="settings-group">
|
||||||
<h2 id="integrations">Integrations</h2>
|
<h2 id="integrations">Integrations</h2>
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
beautifulsoup4==4.10.0
|
beautifulsoup4==4.10.0
|
||||||
celery==5.2.3
|
celery==5.2.3
|
||||||
Django==4.0.1
|
Django==4.0.2
|
||||||
django-cors-headers==3.11.0
|
django-cors-headers==3.11.0
|
||||||
djangorestframework==3.13.1
|
djangorestframework==3.13.1
|
||||||
Pillow==9.0.0
|
Pillow==9.0.1
|
||||||
redis==4.1.1
|
redis==4.1.2
|
||||||
requests==2.27.1
|
requests==2.27.1
|
||||||
ryd-client==0.0.3
|
ryd-client==0.0.3
|
||||||
uWSGI==2.0.20
|
uWSGI==2.0.20
|
||||||
whitenoise==5.3.0
|
whitenoise==5.3.0
|
||||||
yt_dlp==2022.1.21
|
yt_dlp==2022.2.4
|
||||||
|
Loading…
Reference in New Issue
Block a user