Merge branch 'testing'

combine new api endpoints from #151
This commit is contained in:
simon 2022-02-05 18:27:24 +07:00
commit 5b37bd059c
11 changed files with 198 additions and 11 deletions

View File

@ -26,13 +26,13 @@ class ApiBaseView(APIView):
self.response = {"data": False} self.response = {"data": False}
self.status_code = False self.status_code = False
self.context = False self.context = False
self.default_conf = AppConfig().config
def config_builder(self): def config_builder(self):
"""build confic context""" """build confic context"""
default_conf = AppConfig().config
self.context = { self.context = {
"es_url": default_conf["application"]["es_url"], "es_url": self.default_conf["application"]["es_url"],
"es_auth": default_conf["application"]["es_auth"], "es_auth": self.default_conf["application"]["es_auth"],
} }
def get_document(self, document_id): def get_document(self, document_id):
@ -48,6 +48,19 @@ class ApiBaseView(APIView):
self.response["data"] = False self.response["data"] = False
self.status_code = response.status_code self.status_code = response.status_code
def process_keys(self):
"""process keys for frontend"""
all_keys = self.response["data"].keys()
if "media_url" in all_keys:
media_url = self.response["data"]["media_url"]
self.response["data"]["media_url"] = f"/media/{media_url}"
if "vid_thumb_url" in all_keys:
youtube_id = self.response["data"]["youtube_id"]
vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
cache_dir = self.default_conf["application"]["cache_dir"]
new_thumb = f"{cache_dir}/{vid_thumb_url}"
self.response["data"]["vid_thumb_url"] = new_thumb
def get_paginate(self): def get_paginate(self):
"""add pagination detail to response""" """add pagination detail to response"""
self.response["paginate"] = False self.response["paginate"] = False
@ -75,6 +88,7 @@ class VideoApiView(ApiBaseView):
"""get request""" """get request"""
self.config_builder() self.config_builder()
self.get_document(video_id) self.get_document(video_id)
self.process_keys()
return Response(self.response, status=self.status_code) return Response(self.response, status=self.status_code)

View File

@ -23,6 +23,8 @@
"format": false, "format": false,
"add_metadata": false, "add_metadata": false,
"add_thumbnail": false, "add_thumbnail": false,
"subtitle": false,
"subtitle_source": false,
"throttledratelimit": false, "throttledratelimit": false,
"integrate_ryd": false "integrate_ryd": false
}, },

View File

@ -212,6 +212,9 @@ class VideoDownloader:
host_uid = self.config["application"]["HOST_UID"] host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"] host_gid = self.config["application"]["HOST_GID"]
channel_name = clean_string(vid_dict["channel"]["channel_name"]) channel_name = clean_string(vid_dict["channel"]["channel_name"])
if len(channel_name) <= 3:
# fall back to channel id
channel_name = vid_dict["channel"]["channel_id"]
# make archive folder with correct permissions # make archive folder with correct permissions
new_folder = os.path.join(videos, channel_name) new_folder = os.path.join(videos, channel_name)
if not os.path.exists(new_folder): if not os.path.exists(new_folder):

View File

@ -25,6 +25,10 @@
"type": "keyword", "type": "keyword",
"index": false "index": false
}, },
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": { "channel_thumb_url": {
"type": "keyword", "type": "keyword",
"index": false "index": false
@ -84,6 +88,10 @@
"type": "keyword", "type": "keyword",
"index": false "index": false
}, },
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": { "channel_thumb_url": {
"type": "keyword", "type": "keyword",
"index": false "index": false

View File

@ -68,6 +68,12 @@ class ApplicationSettingsForm(forms.Form):
("1", "enable Cast"), ("1", "enable Cast"),
] ]
SUBTITLE_SOURCE_CHOICES = [
("", "-- change subtitle source settings"),
("auto", "also download auto generated"),
("user", "only download uploader"),
]
subscriptions_channel_size = forms.IntegerField(required=False) subscriptions_channel_size = forms.IntegerField(required=False)
downloads_limit_count = forms.IntegerField(required=False) downloads_limit_count = forms.IntegerField(required=False)
downloads_limit_speed = forms.IntegerField(required=False) downloads_limit_speed = forms.IntegerField(required=False)
@ -81,6 +87,10 @@ class ApplicationSettingsForm(forms.Form):
downloads_add_thumbnail = forms.ChoiceField( downloads_add_thumbnail = forms.ChoiceField(
widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False
) )
downloads_subtitle = forms.CharField(required=False)
downloads_subtitle_source = forms.ChoiceField(
widget=forms.Select, choices=SUBTITLE_SOURCE_CHOICES, required=False
)
downloads_integrate_ryd = forms.ChoiceField( downloads_integrate_ryd = forms.ChoiceField(
widget=forms.Select, choices=RYD_CHOICES, required=False widget=forms.Select, choices=RYD_CHOICES, required=False
) )

View File

@ -198,6 +198,9 @@ class YoutubeChannel(YouTubeItem):
"""get folder where media files get stored""" """get folder where media files get stored"""
channel_name = self.json_data["channel_name"] channel_name = self.json_data["channel_name"]
folder_name = clean_string(channel_name) folder_name = clean_string(channel_name)
if len(folder_name) <= 3:
# fall back to channel id
folder_name = self.json_data["channel_id"]
folder_path = os.path.join(self.app_conf["videos"], folder_name) folder_path = os.path.join(self.app_conf["videos"], folder_name)
return folder_path return folder_path

View File

@ -122,6 +122,7 @@ class Pagination:
"page_from": page_from, "page_from": page_from,
"prev_pages": prev_pages, "prev_pages": prev_pages,
"current_page": page_get, "current_page": page_get,
"max_hits": False,
} }
if self.search_get: if self.search_get:
pagination.update({"search_get": self.search_get}) pagination.update({"search_get": self.search_get})
@ -131,6 +132,11 @@ class Pagination:
"""validate pagination with total_hits after making api call""" """validate pagination with total_hits after making api call"""
page_get = self.page_get page_get = self.page_get
max_pages = math.ceil(total_hits / self.page_size) max_pages = math.ceil(total_hits / self.page_size)
if total_hits > 10000:
# es returns maximal 10000 results
self.pagination["max_hits"] = True
max_pages = max_pages - 1
if page_get < max_pages and max_pages > 1: if page_get < max_pages and max_pages > 1:
self.pagination["last_page"] = max_pages self.pagination["last_page"] = max_pages
else: else:

View File

@ -14,7 +14,107 @@ from home.src.ta.helper import DurationConverter, clean_string
from ryd_client import ryd_client from ryd_client import ryd_client
class YoutubeVideo(YouTubeItem): class YoutubeSubtitle:
"""handle video subtitle functionality"""
def __init__(self, config, youtube_meta, media_url, youtube_id):
self.config = config
self.youtube_meta = youtube_meta
self.media_url = media_url
self.youtube_id = youtube_id
self.languages = False
def sub_conf_parse(self):
"""add additional conf values to self"""
languages_raw = self.config["downloads"]["subtitle"]
self.languages = [i.strip() for i in languages_raw.split(",")]
def get_subtitles(self):
"""check what to do"""
self.sub_conf_parse()
if not self.languages:
# no subtitles
return False
relevant_subtitles = self.get_user_subtitles()
if relevant_subtitles:
return relevant_subtitles
if self.config["downloads"]["subtitle_source"] == "auto":
relevant_auto = self.get_auto_caption()
return relevant_auto
return False
def get_auto_caption(self):
"""get auto_caption subtitles"""
print(f"{self.youtube_id}: get auto generated subtitles")
all_subtitles = self.youtube_meta.get("automatic_captions")
if not all_subtitles:
return False
relevant_subtitles = []
for lang in self.languages:
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
all_formats = all_subtitles.get(lang)
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
subtitle.update(
{"lang": lang, "source": "auto", "media_url": media_url}
)
relevant_subtitles.append(subtitle)
break
return relevant_subtitles
def _normalize_lang(self):
"""normalize country specific language keys"""
all_subtitles = self.youtube_meta.get("subtitles")
all_keys = list(all_subtitles.keys())
for key in all_keys:
lang = key.split("-")[0]
old = all_subtitles.pop(key)
all_subtitles[lang] = old
return all_subtitles
def get_user_subtitles(self):
"""get subtitles uploaded from channel owner"""
print(f"{self.youtube_id}: get user uploaded subtitles")
all_subtitles = self._normalize_lang()
if not all_subtitles:
return False
relevant_subtitles = []
for lang in self.languages:
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
all_formats = all_subtitles.get(lang)
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
subtitle.update(
{"lang": lang, "source": "user", "media_url": media_url}
)
relevant_subtitles.append(subtitle)
break
return relevant_subtitles
def download_subtitles(self, relevant_subtitles):
"""download subtitle files to archive"""
for subtitle in relevant_subtitles:
dest_path = os.path.join(
self.config["application"]["videos"], subtitle["media_url"]
)
response = requests.get(subtitle["url"])
if response.ok:
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(response.text)
else:
print(f"{self.youtube_id}: failed to download subtitle")
class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
"""represents a single youtube video""" """represents a single youtube video"""
es_path = False es_path = False
@ -37,6 +137,7 @@ class YoutubeVideo(YouTubeItem):
self._add_stats() self._add_stats()
self.add_file_path() self.add_file_path()
self.add_player() self.add_player()
self._check_subtitles()
if self.config["downloads"]["integrate_ryd"]: if self.config["downloads"]["integrate_ryd"]:
self._get_ryd_stats() self._get_ryd_stats()
@ -96,7 +197,7 @@ class YoutubeVideo(YouTubeItem):
vid_path = os.path.join(cache_path, file_cached) vid_path = os.path.join(cache_path, file_cached)
return vid_path return vid_path
return False raise FileNotFoundError
def add_player(self): def add_player(self):
"""add player information for new videos""" """add player information for new videos"""
@ -125,6 +226,10 @@ class YoutubeVideo(YouTubeItem):
"""build media_url for where file will be located""" """build media_url for where file will be located"""
channel_name = self.json_data["channel"]["channel_name"] channel_name = self.json_data["channel"]["channel_name"]
clean_channel_name = clean_string(channel_name) clean_channel_name = clean_string(channel_name)
if len(clean_channel_name) <= 3:
# fall back to channel id
clean_channel_name = self.json_data["channel"]["channel_id"]
timestamp = self.json_data["published"].replace("-", "") timestamp = self.json_data["published"].replace("-", "")
youtube_id = self.json_data["youtube_id"] youtube_id = self.json_data["youtube_id"]
title = self.json_data["title"] title = self.json_data["title"]
@ -163,6 +268,19 @@ class YoutubeVideo(YouTubeItem):
return True return True
def _check_subtitles(self):
"""optionally add subtitles"""
handler = YoutubeSubtitle(
self.config,
self.youtube_meta,
media_url=self.json_data["media_url"],
youtube_id=self.youtube_id,
)
subtitles = handler.get_subtitles()
if subtitles:
self.json_data["subtitles"] = subtitles
handler.download_subtitles(relevant_subtitles=subtitles)
def index_new_video(youtube_id): def index_new_video(youtube_id):
"""combined classes to create new video in index""" """combined classes to create new video in index"""

View File

@ -109,9 +109,21 @@
{% endif %} {% endif %}
{% if pagination.last_page > 0 %} {% if pagination.last_page > 0 %}
{% if pagination.search_get %} {% if pagination.search_get %}
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">Last ({{ pagination.last_page }})</a> <a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
{% if pagination.max_hits %}
Max ({{ pagination.last_page }})
{% else %}
Last ({{ pagination.last_page }})
{% endif %}
</a>
{% else %} {% else %}
<a class="pagination-item" href="?page={{ pagination.last_page }}">Last ({{ pagination.last_page }})</a> <a class="pagination-item" href="?page={{ pagination.last_page }}">
{% if pagination.max_hits %}
Max ({{ pagination.last_page }})
{% else %}
Last ({{ pagination.last_page }})
{% endif %}
</a>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endif %} {% endif %}

View File

@ -94,6 +94,17 @@
<i>Embed thumbnail into the mediafile.</i><br> <i>Embed thumbnail into the mediafile.</i><br>
{{ app_form.downloads_add_thumbnail }} {{ app_form.downloads_add_thumbnail }}
</div> </div>
<div class="settings-item">
<p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
<i>Choose which subtitles to download, add comma separated two letter language ISO code,<br>
e.g. <span class="settings-current">en, de</span></i><br>
{{ app_form.downloads_subtitle }}</p>
</div>
<div class="settings-item">
<p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
<i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
{{ app_form.downloads_subtitle_source }}
</div>
</div> </div>
<div class="settings-group"> <div class="settings-group">
<h2 id="integrations">Integrations</h2> <h2 id="integrations">Integrations</h2>

View File

@ -1,12 +1,12 @@
beautifulsoup4==4.10.0 beautifulsoup4==4.10.0
celery==5.2.3 celery==5.2.3
Django==4.0.1 Django==4.0.2
django-cors-headers==3.11.0 django-cors-headers==3.11.0
djangorestframework==3.13.1 djangorestframework==3.13.1
Pillow==9.0.0 Pillow==9.0.1
redis==4.1.1 redis==4.1.2
requests==2.27.1 requests==2.27.1
ryd-client==0.0.3 ryd-client==0.0.3
uWSGI==2.0.20 uWSGI==2.0.20
whitenoise==5.3.0 whitenoise==5.3.0
yt_dlp==2022.1.21 yt_dlp==2022.2.4