tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'testing' 

combine new api endpoints from #151
This commit is contained in:
simon 2022-02-05 18:27:24 +07:00
parent 9079a2a78b 8fe00e2152
commit 5b37bd059c
11 changed files with 198 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tubearchivist/api/views.py
View File

@ -26,13 +26,13 @@ class ApiBaseView(APIView):
self.response = {"data": False}
self.status_code = False
self.context = False
self.default_conf = AppConfig().config
def config_builder(self):
"""build confic context"""
default_conf = AppConfig().config
self.context = {
"es_url": default_conf["application"]["es_url"],
"es_auth": default_conf["application"]["es_auth"],
"es_url": self.default_conf["application"]["es_url"],
"es_auth": self.default_conf["application"]["es_auth"],
}
def get_document(self, document_id):
@ -48,6 +48,19 @@ class ApiBaseView(APIView):
self.response["data"] = False
self.status_code = response.status_code
def process_keys(self):
"""process keys for frontend"""
all_keys = self.response["data"].keys()
if "media_url" in all_keys:
media_url = self.response["data"]["media_url"]
self.response["data"]["media_url"] = f"/media/{media_url}"
if "vid_thumb_url" in all_keys:
youtube_id = self.response["data"]["youtube_id"]
vid_thumb_url = ThumbManager().vid_thumb_path(youtube_id)
cache_dir = self.default_conf["application"]["cache_dir"]
new_thumb = f"{cache_dir}/{vid_thumb_url}"
self.response["data"]["vid_thumb_url"] = new_thumb
def get_paginate(self):
"""add pagination detail to response"""
self.response["paginate"] = False
@ -75,6 +88,7 @@ class VideoApiView(ApiBaseView):
"""get request"""
self.config_builder()
self.get_document(video_id)
self.process_keys()
return Response(self.response, status=self.status_code)

2
tubearchivist/home/config.json
View File

@ -23,6 +23,8 @@
"format": false,
"add_metadata": false,
"add_thumbnail": false,
"subtitle": false,
"subtitle_source": false,
"throttledratelimit": false,
"integrate_ryd": false
},

3
tubearchivist/home/src/download/yt_dlp_handler.py
View File

@ -212,6 +212,9 @@ class VideoDownloader:
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
channel_name = clean_string(vid_dict["channel"]["channel_name"])
if len(channel_name) <= 3:
# fall back to channel id
channel_name = vid_dict["channel"]["channel_id"]
# make archive folder with correct permissions
new_folder = os.path.join(videos, channel_name)
if not os.path.exists(new_folder):

8
tubearchivist/home/src/es/index_mapping.json
View File

@ -25,6 +25,10 @@
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false
@ -84,6 +88,10 @@
"type": "keyword",
"index": false
},
"channel_tvart_url": {
"type": "keyword",
"index": false
},
"channel_thumb_url": {
"type": "keyword",
"index": false

10
tubearchivist/home/src/frontend/forms.py
View File

@ -68,6 +68,12 @@ class ApplicationSettingsForm(forms.Form):
("1", "enable Cast"),
]
SUBTITLE_SOURCE_CHOICES = [
("", "-- change subtitle source settings"),
("auto", "also download auto generated"),
("user", "only download uploader"),
]
subscriptions_channel_size = forms.IntegerField(required=False)
downloads_limit_count = forms.IntegerField(required=False)
downloads_limit_speed = forms.IntegerField(required=False)
@ -81,6 +87,10 @@ class ApplicationSettingsForm(forms.Form):
downloads_add_thumbnail = forms.ChoiceField(
widget=forms.Select, choices=THUMBNAIL_CHOICES, required=False
)
downloads_subtitle = forms.CharField(required=False)
downloads_subtitle_source = forms.ChoiceField(
widget=forms.Select, choices=SUBTITLE_SOURCE_CHOICES, required=False
)
downloads_integrate_ryd = forms.ChoiceField(
widget=forms.Select, choices=RYD_CHOICES, required=False
)

3
tubearchivist/home/src/index/channel.py
View File

@ -198,6 +198,9 @@ class YoutubeChannel(YouTubeItem):
"""get folder where media files get stored"""
channel_name = self.json_data["channel_name"]
folder_name = clean_string(channel_name)
if len(folder_name) <= 3:
# fall back to channel id
folder_name = self.json_data["channel_id"]
folder_path = os.path.join(self.app_conf["videos"], folder_name)
return folder_path

6
tubearchivist/home/src/index/generic.py
View File

@ -122,6 +122,7 @@ class Pagination:
"page_from": page_from,
"prev_pages": prev_pages,
"current_page": page_get,
"max_hits": False,
}
if self.search_get:
pagination.update({"search_get": self.search_get})
@ -131,6 +132,11 @@ class Pagination:
"""validate pagination with total_hits after making api call"""
page_get = self.page_get
max_pages = math.ceil(total_hits / self.page_size)
if total_hits > 10000:
# es returns maximal 10000 results
self.pagination["max_hits"] = True
max_pages = max_pages - 1
if page_get < max_pages and max_pages > 1:
self.pagination["last_page"] = max_pages
else:

122
tubearchivist/home/src/index/video.py
View File

@ -14,7 +14,107 @@ from home.src.ta.helper import DurationConverter, clean_string
from ryd_client import ryd_client
class YoutubeVideo(YouTubeItem):
class YoutubeSubtitle:
"""handle video subtitle functionality"""
def __init__(self, config, youtube_meta, media_url, youtube_id):
self.config = config
self.youtube_meta = youtube_meta
self.media_url = media_url
self.youtube_id = youtube_id
self.languages = False
def sub_conf_parse(self):
"""add additional conf values to self"""
languages_raw = self.config["downloads"]["subtitle"]
self.languages = [i.strip() for i in languages_raw.split(",")]
def get_subtitles(self):
"""check what to do"""
self.sub_conf_parse()
if not self.languages:
# no subtitles
return False
relevant_subtitles = self.get_user_subtitles()
if relevant_subtitles:
return relevant_subtitles
if self.config["downloads"]["subtitle_source"] == "auto":
relevant_auto = self.get_auto_caption()
return relevant_auto
return False
def get_auto_caption(self):
"""get auto_caption subtitles"""
print(f"{self.youtube_id}: get auto generated subtitles")
all_subtitles = self.youtube_meta.get("automatic_captions")
if not all_subtitles:
return False
relevant_subtitles = []
for lang in self.languages:
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
all_formats = all_subtitles.get(lang)
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
subtitle.update(
{"lang": lang, "source": "auto", "media_url": media_url}
)
relevant_subtitles.append(subtitle)
break
return relevant_subtitles
def _normalize_lang(self):
"""normalize country specific language keys"""
all_subtitles = self.youtube_meta.get("subtitles")
all_keys = list(all_subtitles.keys())
for key in all_keys:
lang = key.split("-")[0]
old = all_subtitles.pop(key)
all_subtitles[lang] = old
return all_subtitles
def get_user_subtitles(self):
"""get subtitles uploaded from channel owner"""
print(f"{self.youtube_id}: get user uploaded subtitles")
all_subtitles = self._normalize_lang()
if not all_subtitles:
return False
relevant_subtitles = []
for lang in self.languages:
media_url = self.media_url.replace(".mp4", f"-{lang}.vtt")
all_formats = all_subtitles.get(lang)
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
subtitle.update(
{"lang": lang, "source": "user", "media_url": media_url}
)
relevant_subtitles.append(subtitle)
break
return relevant_subtitles
def download_subtitles(self, relevant_subtitles):
"""download subtitle files to archive"""
for subtitle in relevant_subtitles:
dest_path = os.path.join(
self.config["application"]["videos"], subtitle["media_url"]
)
response = requests.get(subtitle["url"])
if response.ok:
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(response.text)
else:
print(f"{self.youtube_id}: failed to download subtitle")
class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
"""represents a single youtube video"""
es_path = False
@ -37,6 +137,7 @@ class YoutubeVideo(YouTubeItem):
self._add_stats()
self.add_file_path()
self.add_player()
self._check_subtitles()
if self.config["downloads"]["integrate_ryd"]:
self._get_ryd_stats()
@ -96,7 +197,7 @@ class YoutubeVideo(YouTubeItem):
vid_path = os.path.join(cache_path, file_cached)
return vid_path
return False
raise FileNotFoundError
def add_player(self):
"""add player information for new videos"""
@ -125,6 +226,10 @@ class YoutubeVideo(YouTubeItem):
"""build media_url for where file will be located"""
channel_name = self.json_data["channel"]["channel_name"]
clean_channel_name = clean_string(channel_name)
if len(clean_channel_name) <= 3:
# fall back to channel id
clean_channel_name = self.json_data["channel"]["channel_id"]
timestamp = self.json_data["published"].replace("-", "")
youtube_id = self.json_data["youtube_id"]
title = self.json_data["title"]
@ -163,6 +268,19 @@ class YoutubeVideo(YouTubeItem):
return True
def _check_subtitles(self):
"""optionally add subtitles"""
handler = YoutubeSubtitle(
self.config,
self.youtube_meta,
media_url=self.json_data["media_url"],
youtube_id=self.youtube_id,
)
subtitles = handler.get_subtitles()
if subtitles:
self.json_data["subtitles"] = subtitles
handler.download_subtitles(relevant_subtitles=subtitles)
def index_new_video(youtube_id):
"""combined classes to create new video in index"""

16
tubearchivist/home/templates/home/base.html
View File

@ -109,9 +109,21 @@
{% endif %}
{% if pagination.last_page > 0 %}
{% if pagination.search_get %}
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">Last ({{ pagination.last_page }})</a>
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
{% if pagination.max_hits %}
Max ({{ pagination.last_page }})
{% else %}
Last ({{ pagination.last_page }})
{% endif %}
</a>
{% else %}
<a class="pagination-item" href="?page={{ pagination.last_page }}">Last ({{ pagination.last_page }})</a>
<a class="pagination-item" href="?page={{ pagination.last_page }}">
{% if pagination.max_hits %}
Max ({{ pagination.last_page }})
{% else %}
Last ({{ pagination.last_page }})
{% endif %}
</a>
{% endif %}
{% endif %}
{% endif %}

11
tubearchivist/home/templates/home/settings.html
View File

@ -94,6 +94,17 @@
<i>Embed thumbnail into the mediafile.</i><br>
{{ app_form.downloads_add_thumbnail }}
</div>
<div class="settings-item">
<p>Subtitles download setting: <span class="settings-current">{{ config.downloads.subtitle }}</span><br>
<i>Choose which subtitles to download, add comma separated two letter language ISO code,<br>
e.g. <span class="settings-current">en, de</span></i><br>
{{ app_form.downloads_subtitle }}</p>
</div>
<div class="settings-item">
<p>Subtitle source settings: <span class="settings-current">{{ config.downloads.subtitle_source }}</span></p>
<i>Download only user generated, or also less accurate auto generated subtitles.</i><br>
{{ app_form.downloads_subtitle_source }}
</div>
</div>
<div class="settings-group">
<h2 id="integrations">Integrations</h2>

8
tubearchivist/requirements.txt
View File

@ -1,12 +1,12 @@
beautifulsoup4==4.10.0
celery==5.2.3
Django==4.0.1
Django==4.0.2
django-cors-headers==3.11.0
djangorestframework==3.13.1
Pillow==9.0.0
redis==4.1.1
Pillow==9.0.1
redis==4.1.2
requests==2.27.1
ryd-client==0.0.3
uWSGI==2.0.20
whitenoise==5.3.0
yt_dlp==2022.1.21
yt_dlp==2022.2.4