mirror of
https://github.com/tubearchivist/tubearchivist-frontend.git
synced 2024-11-22 11:50:14 +00:00
Subtitle parser rewrite, #build
Changes: - merges fix for progress bar issue on player close - rewrite subtitle parser to use json3 - combining 5 cues into single es document for more efficient indexing
This commit is contained in:
commit
34708dd59f
@ -6,7 +6,6 @@ functionality:
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
@ -24,7 +23,7 @@ class YoutubeSubtitle:
|
|||||||
self.video = video
|
self.video = video
|
||||||
self.languages = False
|
self.languages = False
|
||||||
|
|
||||||
def sub_conf_parse(self):
|
def _sub_conf_parse(self):
|
||||||
"""add additional conf values to self"""
|
"""add additional conf values to self"""
|
||||||
languages_raw = self.video.config["downloads"]["subtitle"]
|
languages_raw = self.video.config["downloads"]["subtitle"]
|
||||||
if languages_raw:
|
if languages_raw:
|
||||||
@ -32,26 +31,26 @@ class YoutubeSubtitle:
|
|||||||
|
|
||||||
def get_subtitles(self):
|
def get_subtitles(self):
|
||||||
"""check what to do"""
|
"""check what to do"""
|
||||||
self.sub_conf_parse()
|
self._sub_conf_parse()
|
||||||
if not self.languages:
|
if not self.languages:
|
||||||
# no subtitles
|
# no subtitles
|
||||||
return False
|
return False
|
||||||
|
|
||||||
relevant_subtitles = []
|
relevant_subtitles = []
|
||||||
for lang in self.languages:
|
for lang in self.languages:
|
||||||
user_sub = self.get_user_subtitles(lang)
|
user_sub = self._get_user_subtitles(lang)
|
||||||
if user_sub:
|
if user_sub:
|
||||||
relevant_subtitles.append(user_sub)
|
relevant_subtitles.append(user_sub)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self.video.config["downloads"]["subtitle_source"] == "auto":
|
if self.video.config["downloads"]["subtitle_source"] == "auto":
|
||||||
auto_cap = self.get_auto_caption(lang)
|
auto_cap = self._get_auto_caption(lang)
|
||||||
if auto_cap:
|
if auto_cap:
|
||||||
relevant_subtitles.append(auto_cap)
|
relevant_subtitles.append(auto_cap)
|
||||||
|
|
||||||
return relevant_subtitles
|
return relevant_subtitles
|
||||||
|
|
||||||
def get_auto_caption(self, lang):
|
def _get_auto_caption(self, lang):
|
||||||
"""get auto_caption subtitles"""
|
"""get auto_caption subtitles"""
|
||||||
print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
|
print(f"{self.video.youtube_id}-{lang}: get auto generated subtitles")
|
||||||
all_subtitles = self.video.youtube_meta.get("automatic_captions")
|
all_subtitles = self.video.youtube_meta.get("automatic_captions")
|
||||||
@ -65,7 +64,7 @@ class YoutubeSubtitle:
|
|||||||
if not all_formats:
|
if not all_formats:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
|
subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
|
||||||
subtitle.update(
|
subtitle.update(
|
||||||
{"lang": lang, "source": "auto", "media_url": media_url}
|
{"lang": lang, "source": "auto", "media_url": media_url}
|
||||||
)
|
)
|
||||||
@ -88,7 +87,7 @@ class YoutubeSubtitle:
|
|||||||
|
|
||||||
return all_subtitles
|
return all_subtitles
|
||||||
|
|
||||||
def get_user_subtitles(self, lang):
|
def _get_user_subtitles(self, lang):
|
||||||
"""get subtitles uploaded from channel owner"""
|
"""get subtitles uploaded from channel owner"""
|
||||||
print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
|
print(f"{self.video.youtube_id}-{lang}: get user uploaded subtitles")
|
||||||
all_subtitles = self._normalize_lang()
|
all_subtitles = self._normalize_lang()
|
||||||
@ -102,7 +101,7 @@ class YoutubeSubtitle:
|
|||||||
# no user subtitles found
|
# no user subtitles found
|
||||||
return False
|
return False
|
||||||
|
|
||||||
subtitle = [i for i in all_formats if i["ext"] == "vtt"][0]
|
subtitle = [i for i in all_formats if i["ext"] == "json3"][0]
|
||||||
subtitle.update(
|
subtitle.update(
|
||||||
{"lang": lang, "source": "user", "media_url": media_url}
|
{"lang": lang, "source": "user", "media_url": media_url}
|
||||||
)
|
)
|
||||||
@ -115,12 +114,13 @@ class YoutubeSubtitle:
|
|||||||
for subtitle in relevant_subtitles:
|
for subtitle in relevant_subtitles:
|
||||||
dest_path = os.path.join(videos_base, subtitle["media_url"])
|
dest_path = os.path.join(videos_base, subtitle["media_url"])
|
||||||
source = subtitle["source"]
|
source = subtitle["source"]
|
||||||
|
lang = subtitle.get("lang")
|
||||||
response = requests.get(subtitle["url"])
|
response = requests.get(subtitle["url"])
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(f"{self.video.youtube_id}: failed to download subtitle")
|
print(f"{self.video.youtube_id}: failed to download subtitle")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
parser = SubtitleParser(response.text, subtitle.get("lang"))
|
parser = SubtitleParser(response.text, lang, source)
|
||||||
parser.process()
|
parser.process()
|
||||||
subtitle_str = parser.get_subtitle_str()
|
subtitle_str = parser.get_subtitle_str()
|
||||||
self._write_subtitle_file(dest_path, subtitle_str)
|
self._write_subtitle_file(dest_path, subtitle_str)
|
||||||
@ -145,109 +145,94 @@ class YoutubeSubtitle:
|
|||||||
class SubtitleParser:
|
class SubtitleParser:
|
||||||
"""parse subtitle str from youtube"""
|
"""parse subtitle str from youtube"""
|
||||||
|
|
||||||
time_reg = r"^([0-9]{2}:?){3}\.[0-9]{3} --> ([0-9]{2}:?){3}\.[0-9]{3}"
|
def __init__(self, subtitle_str, lang, source):
|
||||||
stamp_reg = r"<([0-9]{2}:?){3}\.[0-9]{3}>"
|
self.subtitle_raw = json.loads(subtitle_str)
|
||||||
tag_reg = r"</?c>"
|
|
||||||
|
|
||||||
def __init__(self, subtitle_str, lang):
|
|
||||||
self.subtitle_str = subtitle_str
|
|
||||||
self.lang = lang
|
self.lang = lang
|
||||||
self.header = False
|
self.source = source
|
||||||
self.parsed_cue_list = False
|
self.all_cues = False
|
||||||
self.all_text_lines = False
|
|
||||||
self.matched = False
|
|
||||||
|
|
||||||
def process(self):
|
def process(self):
|
||||||
"""collection to process subtitle string"""
|
"""extract relevant que data"""
|
||||||
self._parse_cues()
|
all_events = self.subtitle_raw.get("events")
|
||||||
self._match_text_lines()
|
if self.source == "auto":
|
||||||
self._add_id()
|
all_events = self._flat_auto_caption(all_events)
|
||||||
self._timestamp_check()
|
|
||||||
|
|
||||||
def _parse_cues(self):
|
self.all_cues = []
|
||||||
"""split into cues"""
|
for idx, event in enumerate(all_events):
|
||||||
all_cues = self.subtitle_str.replace("\n \n", "\n").split("\n\n")
|
cue = {
|
||||||
self.header = all_cues[0]
|
"start": self._ms_conv(event["tStartMs"]),
|
||||||
self.all_text_lines = []
|
"end": self._ms_conv(event["tStartMs"] + event["dDurationMs"]),
|
||||||
self.parsed_cue_list = [self._cue_cleaner(i) for i in all_cues[1:]]
|
"text": "".join([i.get("utf8") for i in event["segs"]]),
|
||||||
|
"idx": idx + 1,
|
||||||
|
}
|
||||||
|
self.all_cues.append(cue)
|
||||||
|
|
||||||
def _cue_cleaner(self, cue):
|
@staticmethod
|
||||||
"""parse single cue"""
|
def _flat_auto_caption(all_events):
|
||||||
all_lines = cue.split("\n")
|
"""flatten autocaption segments"""
|
||||||
cue_dict = {"lines": []}
|
flatten = []
|
||||||
|
for event in all_events:
|
||||||
for line in all_lines:
|
if "segs" not in event.keys():
|
||||||
if re.match(self.time_reg, line):
|
continue
|
||||||
clean = re.search(self.time_reg, line).group()
|
text = "".join([i.get("utf8") for i in event.get("segs")])
|
||||||
start, end = clean.split(" --> ")
|
if not text.strip():
|
||||||
cue_dict.update({"start": start, "end": end})
|
|
||||||
else:
|
|
||||||
clean = re.sub(self.stamp_reg, "", line)
|
|
||||||
clean = re.sub(self.tag_reg, "", clean)
|
|
||||||
cue_dict["lines"].append(clean)
|
|
||||||
if clean.strip() and clean not in self.all_text_lines[-4:]:
|
|
||||||
# remove immediate duplicates
|
|
||||||
self.all_text_lines.append(clean)
|
|
||||||
|
|
||||||
return cue_dict
|
|
||||||
|
|
||||||
def _match_text_lines(self):
|
|
||||||
"""match unique text lines with timestamps"""
|
|
||||||
|
|
||||||
self.matched = []
|
|
||||||
|
|
||||||
while self.all_text_lines:
|
|
||||||
check = self.all_text_lines[0]
|
|
||||||
matches = [i for i in self.parsed_cue_list if check in i["lines"]]
|
|
||||||
new_cue = matches[-1]
|
|
||||||
new_cue["start"] = matches[0]["start"]
|
|
||||||
|
|
||||||
for line in new_cue["lines"]:
|
|
||||||
try:
|
|
||||||
self.all_text_lines.remove(line)
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.matched.append(new_cue)
|
|
||||||
|
|
||||||
def _timestamp_check(self):
|
|
||||||
"""check if end timestamp is bigger than start timestamp"""
|
|
||||||
for idx, cue in enumerate(self.matched):
|
|
||||||
# this
|
|
||||||
end = int(re.sub("[^0-9]", "", cue.get("end")))
|
|
||||||
# next
|
|
||||||
try:
|
|
||||||
next_cue = self.matched[idx + 1]
|
|
||||||
except IndexError:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
start_next = int(re.sub("[^0-9]", "", next_cue.get("start")))
|
if flatten:
|
||||||
if end > start_next:
|
# fix overlapping retiming issue
|
||||||
self.matched[idx]["end"] = next_cue.get("start")
|
last_end = flatten[-1]["tStartMs"] + flatten[-1]["dDurationMs"]
|
||||||
|
if event["tStartMs"] < last_end:
|
||||||
|
joined = flatten[-1]["segs"][0]["utf8"] + "\n" + text
|
||||||
|
flatten[-1]["segs"][0]["utf8"] = joined
|
||||||
|
continue
|
||||||
|
|
||||||
def _add_id(self):
|
event.update({"segs": [{"utf8": text}]})
|
||||||
"""add id to matched cues"""
|
flatten.append(event)
|
||||||
for idx, _ in enumerate(self.matched):
|
|
||||||
self.matched[idx]["id"] = idx + 1
|
return flatten
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ms_conv(ms):
|
||||||
|
"""convert ms to timestamp"""
|
||||||
|
hours = str((ms // (1000 * 60 * 60)) % 24).zfill(2)
|
||||||
|
minutes = str((ms // (1000 * 60)) % 60).zfill(2)
|
||||||
|
secs = str((ms // 1000) % 60).zfill(2)
|
||||||
|
millis = str(ms % 1000).zfill(3)
|
||||||
|
|
||||||
|
return f"{hours}:{minutes}:{secs}.{millis}"
|
||||||
|
|
||||||
def get_subtitle_str(self):
|
def get_subtitle_str(self):
|
||||||
"""stitch cues and return processed new string"""
|
"""create vtt text str from cues"""
|
||||||
new_subtitle_str = self.header + "\n\n"
|
subtitle_str = f"WEBVTT\nKind: captions\nLanguage: {self.lang}"
|
||||||
|
|
||||||
for cue in self.matched:
|
for cue in self.all_cues:
|
||||||
timestamp = f"{cue.get('start')} --> {cue.get('end')}"
|
stamp = f"{cue.get('start')} --> {cue.get('end')}"
|
||||||
lines = "\n".join(cue.get("lines"))
|
cue_text = f"\n\n{cue.get('idx')}\n{stamp}\n{cue.get('text')}"
|
||||||
cue_text = f"{cue.get('id')}\n{timestamp}\n{lines}\n\n"
|
subtitle_str = subtitle_str + cue_text
|
||||||
new_subtitle_str = new_subtitle_str + cue_text
|
|
||||||
|
|
||||||
return new_subtitle_str
|
return subtitle_str
|
||||||
|
|
||||||
def create_bulk_import(self, video, source):
|
def create_bulk_import(self, video, source):
|
||||||
"""process matched for es import"""
|
"""subtitle lines for es import"""
|
||||||
|
documents = self._create_documents(video, source)
|
||||||
bulk_list = []
|
bulk_list = []
|
||||||
channel = video.json_data.get("channel")
|
|
||||||
|
|
||||||
document = {
|
for document in documents:
|
||||||
|
document_id = document.get("subtitle_fragment_id")
|
||||||
|
action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
|
||||||
|
bulk_list.append(json.dumps(action))
|
||||||
|
bulk_list.append(json.dumps(document))
|
||||||
|
|
||||||
|
bulk_list.append("\n")
|
||||||
|
query_str = "\n".join(bulk_list)
|
||||||
|
|
||||||
|
return query_str
|
||||||
|
|
||||||
|
def _create_documents(self, video, source):
|
||||||
|
"""process documents"""
|
||||||
|
documents = self._chunk_list(video.youtube_id)
|
||||||
|
channel = video.json_data.get("channel")
|
||||||
|
meta_dict = {
|
||||||
"youtube_id": video.youtube_id,
|
"youtube_id": video.youtube_id,
|
||||||
"title": video.json_data.get("title"),
|
"title": video.json_data.get("title"),
|
||||||
"subtitle_channel": channel.get("channel_name"),
|
"subtitle_channel": channel.get("channel_name"),
|
||||||
@ -257,26 +242,35 @@ class SubtitleParser:
|
|||||||
"subtitle_source": source,
|
"subtitle_source": source,
|
||||||
}
|
}
|
||||||
|
|
||||||
for match in self.matched:
|
_ = [i.update(meta_dict) for i in documents]
|
||||||
match_id = match.get("id")
|
|
||||||
document_id = f"{video.youtube_id}-{self.lang}-{match_id}"
|
return documents
|
||||||
action = {"index": {"_index": "ta_subtitle", "_id": document_id}}
|
|
||||||
document.update(
|
def _chunk_list(self, youtube_id):
|
||||||
{
|
"""join cues for bulk import"""
|
||||||
"subtitle_fragment_id": document_id,
|
chunk_list = []
|
||||||
"subtitle_start": match.get("start"),
|
|
||||||
"subtitle_end": match.get("end"),
|
chunk = {}
|
||||||
"subtitle_index": match_id,
|
for cue in self.all_cues:
|
||||||
"subtitle_line": " ".join(match.get("lines")),
|
if chunk:
|
||||||
|
text = f"{chunk.get('subtitle_line')} {cue.get('text')}\n"
|
||||||
|
chunk["subtitle_line"] = text
|
||||||
|
else:
|
||||||
|
idx = len(chunk_list) + 1
|
||||||
|
chunk = {
|
||||||
|
"subtitle_index": idx,
|
||||||
|
"subtitle_line": cue.get("text"),
|
||||||
|
"subtitle_start": cue.get("start"),
|
||||||
}
|
}
|
||||||
)
|
|
||||||
bulk_list.append(json.dumps(action))
|
|
||||||
bulk_list.append(json.dumps(document))
|
|
||||||
|
|
||||||
bulk_list.append("\n")
|
chunk["subtitle_fragment_id"] = f"{youtube_id}-{self.lang}-{idx}"
|
||||||
query_str = "\n".join(bulk_list)
|
|
||||||
|
|
||||||
return query_str
|
if cue["idx"] % 5 == 0:
|
||||||
|
chunk["subtitle_end"] = cue.get("end")
|
||||||
|
chunk_list.append(chunk)
|
||||||
|
chunk = {}
|
||||||
|
|
||||||
|
return chunk_list
|
||||||
|
|
||||||
|
|
||||||
class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
|
||||||
|
@ -124,9 +124,9 @@
|
|||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.source.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="isUnwatched(this.id)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="isWatched(this.id)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
|
@ -9,14 +9,14 @@
|
|||||||
<div class="video-list {{ view_style }}">
|
<div class="video-list {{ view_style }}">
|
||||||
{% for video in continue_vids %}
|
{% for video in continue_vids %}
|
||||||
<div class="video-item {{ view_style }}">
|
<div class="video-item {{ view_style }}">
|
||||||
<a href="#player" data-id="{{ video.youtube_id }}" onclick="createPlayer(this)">
|
<a href="#player" data-id="{{ video.source.youtube_id }}" onclick="createPlayer(this)">
|
||||||
<div class="video-thumb-wrap {{ view_style }}">
|
<div class="video-thumb-wrap {{ view_style }}">
|
||||||
<div class="video-thumb">
|
<div class="video-thumb">
|
||||||
<img src="/cache/{{ video.vid_thumb_url }}" alt="video-thumb">
|
<img src="/cache/{{ video.source.vid_thumb_url }}" alt="video-thumb">
|
||||||
{% if video.player.progress %}
|
{% if video.source.player.progress %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: {{video.player.progress}}%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: {{video.source.player.progress}}%;"></div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="video-progress-bar" id="progress-{{ video.youtube_id }}" style="width: 0%;"></div>
|
<div class="video-progress-bar" id="progress-{{ video.source.youtube_id }}" style="width: 0%;"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="video-play">
|
<div class="video-play">
|
||||||
@ -25,17 +25,17 @@
|
|||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||||
{% if video.player.watched %}
|
{% if video.source.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="isUnwatched(this.id)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="isWatched(this.id)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.published }} | {{ video.player.duration_str }}</span>
|
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<a href="{% url 'channel_id' video.channel.channel_id %}"><h3>{{ video.channel.channel_name }}</h3></a>
|
<a href="{% url 'channel_id' video.source.channel.channel_id %}"><h3>{{ video.source.channel.channel_name }}</h3></a>
|
||||||
<a class="video-more" href="{% url 'video' video.youtube_id %}"><h2>{{ video.title }}</h2></a>
|
<a class="video-more" href="{% url 'video' video.source.youtube_id %}"><h2>{{ video.source.title }}</h2></a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@ -103,9 +103,9 @@
|
|||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.source.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="isUnwatched(this.id)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="isWatched(this.id)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
|
@ -105,9 +105,9 @@
|
|||||||
<div class="video-desc {{ view_style }}">
|
<div class="video-desc {{ view_style }}">
|
||||||
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
<div class="video-desc-player" id="video-info-{{ video.source.youtube_id }}">
|
||||||
{% if video.source.player.watched %}
|
{% if video.source.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="isUnwatched(this.id)" class="watch-button" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.source.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="isWatched(this.id)" class="watch-button" title="Mark as watched">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.source.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
<span>{{ video.source.published }} | {{ video.source.player.duration_str }}</span>
|
||||||
</div>
|
</div>
|
||||||
|
@ -32,9 +32,9 @@
|
|||||||
<p>Last refreshed: {{ video.vid_last_refresh }}</p>
|
<p>Last refreshed: {{ video.vid_last_refresh }}</p>
|
||||||
<p class="video-info-watched">Watched:
|
<p class="video-info-watched">Watched:
|
||||||
{% if video.player.watched %}
|
{% if video.player.watched %}
|
||||||
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" id="{{ video.youtube_id }}" onclick="isUnwatched(this.id)" class="seen-icon" title="Mark as unwatched">
|
<img src="{% static 'img/icon-seen.svg' %}" alt="seen-icon" data-id="{{ video.youtube_id }}" data-status="watched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as unwatched">
|
||||||
{% else %}
|
{% else %}
|
||||||
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" id="{{ video.youtube_id }}" onclick="isWatched(this.id)" class="unseen-icon" title="Mark as watched.">
|
<img src="{% static 'img/icon-unseen.svg' %}" alt="unseen-icon" data-id="{{ video.youtube_id }}" data-status="unwatched" onclick="updateVideoWatchStatus(this)" class="watch-button" title="Mark as watched">
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</p>
|
</p>
|
||||||
{% if video.active %}
|
{% if video.active %}
|
||||||
|
@ -175,15 +175,36 @@ class ArchivistResultsView(ArchivistViewConfig):
|
|||||||
if not results or not self.context["results"]:
|
if not results or not self.context["results"]:
|
||||||
return
|
return
|
||||||
|
|
||||||
self.context["continue_vids"] = []
|
self.context["continue_vids"] = self.get_in_progress(results)
|
||||||
progress = {i["youtube_id"]: i["position"] for i in results}
|
|
||||||
|
in_progress = {i["youtube_id"]: i["position"] for i in results}
|
||||||
for hit in self.context["results"]:
|
for hit in self.context["results"]:
|
||||||
video = hit["source"]
|
video = hit["source"]
|
||||||
if video["youtube_id"] in progress:
|
if video["youtube_id"] in in_progress:
|
||||||
played_sec = progress.get(video["youtube_id"])
|
played_sec = in_progress.get(video["youtube_id"])
|
||||||
total = video["player"]["duration"]
|
total = video["player"]["duration"]
|
||||||
video["player"]["progress"] = 100 * (played_sec / total)
|
video["player"]["progress"] = 100 * (played_sec / total)
|
||||||
self.context["continue_vids"].append(video)
|
|
||||||
|
def get_in_progress(self, results):
|
||||||
|
"""get all videos in progress"""
|
||||||
|
ids = [{"match": {"youtube_id": i.get("youtube_id")}} for i in results]
|
||||||
|
data = {
|
||||||
|
"size": self.default_conf["archive"]["page_size"],
|
||||||
|
"query": {"bool": {"should": ids}},
|
||||||
|
"sort": [{"published": {"order": "desc"}}],
|
||||||
|
}
|
||||||
|
search = SearchHandler(
|
||||||
|
"ta_video/_search", self.default_conf, data=data
|
||||||
|
)
|
||||||
|
videos = search.get_data()
|
||||||
|
for video in videos:
|
||||||
|
youtube_id = video["source"]["youtube_id"]
|
||||||
|
matched = [i for i in results if i["youtube_id"] == youtube_id]
|
||||||
|
played_sec = matched[0]["position"]
|
||||||
|
total = video["source"]["player"]["duration"]
|
||||||
|
video["source"]["player"]["progress"] = 100 * (played_sec / total)
|
||||||
|
|
||||||
|
return videos
|
||||||
|
|
||||||
def single_lookup(self, es_path):
|
def single_lookup(self, es_path):
|
||||||
"""retrieve a single item from url"""
|
"""retrieve a single item from url"""
|
||||||
|
@ -9,4 +9,4 @@ requests==2.27.1
|
|||||||
ryd-client==0.0.3
|
ryd-client==0.0.3
|
||||||
uWSGI==2.0.20
|
uWSGI==2.0.20
|
||||||
whitenoise==6.0.0
|
whitenoise==6.0.0
|
||||||
yt_dlp==2022.2.4
|
yt_dlp==2022.3.8.2
|
||||||
|
@ -8,21 +8,62 @@ function sortChange(sortValue) {
|
|||||||
}, 500);
|
}, 500);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWatched(youtube_id) {
|
// Updates video watch status when passed a video id and it's current state (ex if the video was unwatched but you want to mark it as watched you will pass "unwatched")
|
||||||
postVideoProgress(youtube_id, 0); // Reset video progress on watched;
|
function updateVideoWatchStatus(input1, videoCurrentWatchStatus) {
|
||||||
removeProgressBar(youtube_id);
|
if (videoCurrentWatchStatus) {
|
||||||
var payload = JSON.stringify({'watched': youtube_id});
|
videoId = input1;
|
||||||
sendPost(payload);
|
} else if (input1.getAttribute("data-id")) {
|
||||||
var seenIcon = document.createElement('img');
|
videoId = input1.getAttribute("data-id");
|
||||||
seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
|
videoCurrentWatchStatus = input1.getAttribute("data-status");
|
||||||
seenIcon.setAttribute('alt', 'seen-icon');
|
}
|
||||||
seenIcon.setAttribute('id', youtube_id);
|
|
||||||
seenIcon.setAttribute('title', "Mark as unwatched");
|
postVideoProgress(videoId, 0); // Reset video progress on watched/unwatched;
|
||||||
seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
|
removeProgressBar(videoId);
|
||||||
seenIcon.classList = 'seen-icon';
|
|
||||||
document.getElementById(youtube_id).replaceWith(seenIcon);
|
if (videoCurrentWatchStatus == "watched") {
|
||||||
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
|
||||||
|
var payload = JSON.stringify({'un_watched': videoId});
|
||||||
|
sendPost(payload);
|
||||||
|
} else if (videoCurrentWatchStatus == "unwatched") {
|
||||||
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
|
||||||
|
var payload = JSON.stringify({'watched': videoId});
|
||||||
|
sendPost(payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
var watchButtons = document.getElementsByClassName("watch-button");
|
||||||
|
for (let i = 0; i < watchButtons.length; i++) {
|
||||||
|
if (watchButtons[i].getAttribute("data-id") == videoId) {
|
||||||
|
watchButtons[i].outerHTML = watchStatusIndicator;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Creates a watch status indicator when passed a video id and the videos watch status
|
||||||
|
function createWatchStatusIndicator(videoId, videoWatchStatus) {
|
||||||
|
if (videoWatchStatus == "watched") {
|
||||||
|
var seen = "seen";
|
||||||
|
var title = "Mark as unwatched";
|
||||||
|
} else if (videoWatchStatus == "unwatched") {
|
||||||
|
var seen = "unseen";
|
||||||
|
var title = "Mark as watched";
|
||||||
|
}
|
||||||
|
var watchStatusIndicator = `<img src="/static/img/icon-${seen}.svg" alt="${seen}-icon" data-id="${videoId}" data-status="${videoWatchStatus}" onclick="updateVideoWatchStatus(this)" class="watch-button" title="${title}">`;
|
||||||
|
return watchStatusIndicator;
|
||||||
|
}
|
||||||
|
|
||||||
|
// function isWatched(youtube_id) {
|
||||||
|
// var payload = JSON.stringify({'watched': youtube_id});
|
||||||
|
// sendPost(payload);
|
||||||
|
// var seenIcon = document.createElement('img');
|
||||||
|
// seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
|
||||||
|
// seenIcon.setAttribute('alt', 'seen-icon');
|
||||||
|
// seenIcon.setAttribute('id', youtube_id);
|
||||||
|
// seenIcon.setAttribute('title', "Mark as unwatched");
|
||||||
|
// seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
|
||||||
|
// seenIcon.classList = 'seen-icon';
|
||||||
|
// document.getElementById(youtube_id).replaceWith(seenIcon);
|
||||||
|
// }
|
||||||
|
|
||||||
// Removes the progress bar when passed a video id
|
// Removes the progress bar when passed a video id
|
||||||
function removeProgressBar(videoId) {
|
function removeProgressBar(videoId) {
|
||||||
setProgressBar(videoId, 0, 1);
|
setProgressBar(videoId, 0, 1);
|
||||||
@ -39,19 +80,19 @@ function isWatchedButton(button) {
|
|||||||
}, 1000);
|
}, 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isUnwatched(youtube_id) {
|
// function isUnwatched(youtube_id) {
|
||||||
postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
|
// postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
|
||||||
var payload = JSON.stringify({'un_watched': youtube_id});
|
// var payload = JSON.stringify({'un_watched': youtube_id});
|
||||||
sendPost(payload);
|
// sendPost(payload);
|
||||||
var unseenIcon = document.createElement('img');
|
// var unseenIcon = document.createElement('img');
|
||||||
unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
|
// unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
|
||||||
unseenIcon.setAttribute('alt', 'unseen-icon');
|
// unseenIcon.setAttribute('alt', 'unseen-icon');
|
||||||
unseenIcon.setAttribute('id', youtube_id);
|
// unseenIcon.setAttribute('id', youtube_id);
|
||||||
unseenIcon.setAttribute('title', "Mark as watched");
|
// unseenIcon.setAttribute('title', "Mark as watched");
|
||||||
unseenIcon.setAttribute('onclick', "isWatched(this.id)");
|
// unseenIcon.setAttribute('onclick', "isWatched(this.id)");
|
||||||
unseenIcon.classList = 'unseen-icon';
|
// unseenIcon.classList = 'unseen-icon';
|
||||||
document.getElementById(youtube_id).replaceWith(unseenIcon);
|
// document.getElementById(youtube_id).replaceWith(unseenIcon);
|
||||||
}
|
// }
|
||||||
|
|
||||||
function unsubscribe(id_unsub) {
|
function unsubscribe(id_unsub) {
|
||||||
var payload = JSON.stringify({'unsubscribe': id_unsub});
|
var payload = JSON.stringify({'unsubscribe': id_unsub});
|
||||||
@ -327,7 +368,7 @@ function createPlayer(button) {
|
|||||||
var channelName = videoData.data.channel.channel_name;
|
var channelName = videoData.data.channel.channel_name;
|
||||||
|
|
||||||
removePlayer();
|
removePlayer();
|
||||||
document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
|
// document.getElementById(videoId).outerHTML = ''; // Remove watch indicator from video info
|
||||||
|
|
||||||
// If cast integration is enabled create cast button
|
// If cast integration is enabled create cast button
|
||||||
var castButton = '';
|
var castButton = '';
|
||||||
@ -337,13 +378,12 @@ function createPlayer(button) {
|
|||||||
|
|
||||||
// Watched indicator
|
// Watched indicator
|
||||||
if (videoData.data.player.watched) {
|
if (videoData.data.player.watched) {
|
||||||
var playerState = "seen";
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
|
||||||
var watchedFunction = "Unwatched";
|
|
||||||
} else {
|
} else {
|
||||||
var playerState = "unseen";
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
|
||||||
var watchedFunction = "Watched";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
var playerStats = `<div class="thumb-icon player-stats"><img src="/static/img/icon-eye.svg" alt="views icon"><span>${videoViews}</span>`;
|
var playerStats = `<div class="thumb-icon player-stats"><img src="/static/img/icon-eye.svg" alt="views icon"><span>${videoViews}</span>`;
|
||||||
if (videoData.data.stats.like_count) {
|
if (videoData.data.stats.like_count) {
|
||||||
var likes = formatNumbers(videoData.data.stats.like_count);
|
var likes = formatNumbers(videoData.data.stats.like_count);
|
||||||
@ -360,7 +400,7 @@ function createPlayer(button) {
|
|||||||
${videoTag}
|
${videoTag}
|
||||||
<div class="player-title boxed-content">
|
<div class="player-title boxed-content">
|
||||||
<img class="close-button" src="/static/img/icon-close.svg" alt="close-icon" data="${videoId}" onclick="removePlayer()" title="Close player">
|
<img class="close-button" src="/static/img/icon-close.svg" alt="close-icon" data="${videoId}" onclick="removePlayer()" title="Close player">
|
||||||
<img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="is${watchedFunction}(this.id)" class="${playerState}-icon" title="Mark as ${watchedFunction}">
|
${watchStatusIndicator}
|
||||||
${castButton}
|
${castButton}
|
||||||
${playerStats}
|
${playerStats}
|
||||||
<div class="player-channel-playlist">
|
<div class="player-channel-playlist">
|
||||||
@ -444,8 +484,12 @@ function getVideoPlayerDuration() {
|
|||||||
function getVideoPlayerWatchStatus() {
|
function getVideoPlayerWatchStatus() {
|
||||||
var videoId = getVideoPlayerVideoId();
|
var videoId = getVideoPlayerVideoId();
|
||||||
var watched = false;
|
var watched = false;
|
||||||
if(document.getElementById(videoId) != null && document.getElementById(videoId).className != "unseen-icon") {
|
|
||||||
watched = true;
|
var watchButtons = document.getElementsByClassName("watch-button");
|
||||||
|
for (let i = 0; i < watchButtons.length; i++) {
|
||||||
|
if (watchButtons[i].getAttribute("data-id") == videoId && watchButtons[i].getAttribute("data-status") == "watched") {
|
||||||
|
watched = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return watched;
|
return watched;
|
||||||
}
|
}
|
||||||
@ -459,7 +503,7 @@ function onVideoProgress() {
|
|||||||
postVideoProgress(videoId, currentTime);
|
postVideoProgress(videoId, currentTime);
|
||||||
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
|
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
|
||||||
if (watchedThreshold(currentTime, duration)) {
|
if (watchedThreshold(currentTime, duration)) {
|
||||||
isWatched(videoId);
|
updateVideoWatchStatus(videoId, "unwatched");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -469,7 +513,7 @@ function onVideoProgress() {
|
|||||||
function onVideoEnded() {
|
function onVideoEnded() {
|
||||||
var videoId = getVideoPlayerVideoId();
|
var videoId = getVideoPlayerVideoId();
|
||||||
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
|
if (!getVideoPlayerWatchStatus()) { // Check if video is already marked as watched
|
||||||
isWatched(videoId);
|
updateVideoWatchStatus(videoId, "unwatched");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -606,13 +650,21 @@ function removePlayer() {
|
|||||||
|
|
||||||
// Sets the progress bar when passed a video id, video progress and video duration
|
// Sets the progress bar when passed a video id, video progress and video duration
|
||||||
function setProgressBar(videoId, currentTime, duration) {
|
function setProgressBar(videoId, currentTime, duration) {
|
||||||
progressBar = document.getElementById("progress-" + videoId);
|
var progressBarWidth = (currentTime / duration) * 100 + "%";
|
||||||
progressBarWidth = (currentTime / duration) * 100 + "%";
|
var progressBars = document.getElementsByClassName("video-progress-bar");
|
||||||
if (progressBar && !getVideoPlayerWatchStatus()) {
|
for (let i = 0; i < progressBars.length; i++) {
|
||||||
progressBar.style.width = progressBarWidth;
|
if (progressBars[i].id == "progress-" + videoId) {
|
||||||
} else if (progressBar) {
|
if (!getVideoPlayerWatchStatus()) {
|
||||||
progressBar.style.width = "0%";
|
progressBars[i].style.width = progressBarWidth;
|
||||||
|
} else {
|
||||||
|
progressBars[i].style.width = "0%";
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// progressBar = document.getElementById("progress-" + videoId);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// multi search form
|
// multi search form
|
||||||
@ -681,9 +733,9 @@ function createVideo(video, viewStyle) {
|
|||||||
const videoPublished = video.published;
|
const videoPublished = video.published;
|
||||||
const videoDuration = video.player.duration_str;
|
const videoDuration = video.player.duration_str;
|
||||||
if (video.player.watched) {
|
if (video.player.watched) {
|
||||||
var playerState = "seen";
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "watched");
|
||||||
} else {
|
} else {
|
||||||
var playerState = "unseen";
|
var watchStatusIndicator = createWatchStatusIndicator(videoId, "unwatched");
|
||||||
};
|
};
|
||||||
const channelId = video.channel.channel_id;
|
const channelId = video.channel.channel_id;
|
||||||
const channelName = video.channel.channel_name;
|
const channelName = video.channel.channel_name;
|
||||||
@ -701,7 +753,7 @@ function createVideo(video, viewStyle) {
|
|||||||
</a>
|
</a>
|
||||||
<div class="video-desc ${viewStyle}">
|
<div class="video-desc ${viewStyle}">
|
||||||
<div class="video-desc-player" id="video-info-${videoId}">
|
<div class="video-desc-player" id="video-info-${videoId}">
|
||||||
<img src="/static/img/icon-${playerState}.svg" alt="${playerState}-icon" id="${videoId}" onclick="isWatched(this.id)" class="${playerState}-icon">
|
${watchStatusIndicator}
|
||||||
<span>${videoPublished} | ${videoDuration}</span>
|
<span>${videoPublished} | ${videoDuration}</span>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
|
Loading…
Reference in New Issue
Block a user