mirror of
https://github.com/tubearchivist/tubearchivist.git
synced 2024-11-10 13:50:13 +00:00
Fixing permissions, squashing bugs #build
Changed: - Add recent videos to reindes queue - Fix channel tv art fallback - Fix notification callback notification off by one - Fix missing reindex notification in frontend - Change new channel folder to honoring UID/GID - Change new subtitles to honour UID/GID - Change manual import to honour UID/GID
This commit is contained in:
commit
5e841bf7f8
@ -81,7 +81,7 @@ class ThumbManagerBase:
|
||||
"banner": os.path.join(
|
||||
app_root, "static/img/default-channel-banner.jpg"
|
||||
),
|
||||
"art": os.path.join(
|
||||
"tvart": os.path.join(
|
||||
app_root, "static/img/default-channel-art.jpg"
|
||||
),
|
||||
}
|
||||
@ -270,6 +270,7 @@ class ValidatorCallback:
|
||||
urls = (
|
||||
channel["_source"]["channel_thumb_url"],
|
||||
channel["_source"]["channel_banner_url"],
|
||||
channel["_source"]["channel_tvart_url"],
|
||||
)
|
||||
handler = ThumbManager(channel["_source"]["channel_id"])
|
||||
handler.download_channel_art(urls, skip_existing=True)
|
||||
|
||||
@ -395,8 +395,8 @@ class VideoDownloader:
|
||||
new_folder = os.path.join(videos, channel_name)
|
||||
if not os.path.exists(new_folder):
|
||||
os.makedirs(new_folder)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_folder, host_uid, host_gid)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_folder, host_uid, host_gid)
|
||||
# find real filename
|
||||
cache_dir = self.config["application"]["cache_dir"]
|
||||
all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))
|
||||
|
||||
@ -166,7 +166,7 @@ class IndexPaginate:
|
||||
def _notify(self, processed):
|
||||
"""send notification on task"""
|
||||
total = self.kwargs.get("total")
|
||||
progress = (processed + 1) / total
|
||||
progress = processed / total
|
||||
index_clean = self.index_name.lstrip("ta_").title()
|
||||
message = [f"Processing {index_clean}s {processed}/{total}"]
|
||||
self.kwargs.get("task").send_progress(message, progress=progress)
|
||||
|
||||
@ -12,6 +12,7 @@ import shutil
|
||||
import subprocess
|
||||
|
||||
from home.src.download.thumbnails import ThumbManager
|
||||
from home.src.index.comments import CommentList
|
||||
from home.src.index.video import YoutubeVideo
|
||||
from home.src.ta.config import AppConfig
|
||||
from home.src.ta.helper import ignore_filelist
|
||||
@ -141,6 +142,9 @@ class ImportFolderScanner:
|
||||
|
||||
ManualImport(current_video, self.CONFIG).run()
|
||||
|
||||
video_ids = [i["video_id"] for i in self.to_import]
|
||||
CommentList(video_ids, task=self.task).index()
|
||||
|
||||
def _notify(self, idx, current_video):
|
||||
"""send notification back to task"""
|
||||
filename = os.path.split(current_video["media"])[-1]
|
||||
@ -427,15 +431,22 @@ class ManualImport:
|
||||
def _move_to_archive(self, json_data):
|
||||
"""move identified media file to archive"""
|
||||
videos = self.config["application"]["videos"]
|
||||
host_uid = self.config["application"]["HOST_UID"]
|
||||
host_gid = self.config["application"]["HOST_GID"]
|
||||
|
||||
channel, file = os.path.split(json_data["media_url"])
|
||||
channel_folder = os.path.join(videos, channel)
|
||||
if not os.path.exists(channel_folder):
|
||||
os.makedirs(channel_folder)
|
||||
|
||||
if host_uid and host_gid:
|
||||
os.chown(channel_folder, host_uid, host_gid)
|
||||
|
||||
old_path = self.current_video["media"]
|
||||
new_path = os.path.join(channel_folder, file)
|
||||
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
|
||||
if host_uid and host_gid:
|
||||
os.chown(new_path, host_uid, host_gid)
|
||||
|
||||
base_name, _ = os.path.splitext(new_path)
|
||||
for old_path in self.current_video["subtitle"]:
|
||||
|
||||
@ -49,6 +49,7 @@ class ReindexBase:
|
||||
}
|
||||
|
||||
MULTIPLY = 1.2
|
||||
DAYS3 = 60 * 60 * 24 * 3
|
||||
|
||||
def __init__(self):
|
||||
self.config = AppConfig().config
|
||||
@ -62,13 +63,34 @@ class ReindexBase:
|
||||
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
|
||||
|
||||
|
||||
class ReindexOutdated(ReindexBase):
|
||||
"""add outdated documents to reindex queue"""
|
||||
class ReindexPopulate(ReindexBase):
|
||||
"""add outdated and recent documents to reindex queue"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.interval = self.config["scheduler"]["check_reindex_days"]
|
||||
|
||||
def add_recent(self):
|
||||
"""add recent videos to refresh"""
|
||||
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
|
||||
must_list = [
|
||||
{"term": {"active": {"value": True}}},
|
||||
{"range": {"published": {"gte": gte}}},
|
||||
]
|
||||
data = {
|
||||
"size": 10000,
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"published": {"order": "desc"}}],
|
||||
}
|
||||
response, _ = ElasticWrap("ta_video/_search").get(data=data)
|
||||
hits = response["hits"]["hits"]
|
||||
if not hits:
|
||||
return
|
||||
|
||||
all_ids = [i["_source"]["youtube_id"] for i in hits]
|
||||
reindex_config = self.REINDEX_CONFIG.get("video")
|
||||
self.populate(all_ids, reindex_config)
|
||||
|
||||
def add_outdated(self):
|
||||
"""add outdated documents"""
|
||||
for reindex_config in self.REINDEX_CONFIG.values():
|
||||
|
||||
@ -115,7 +115,7 @@ class YoutubeSubtitle:
|
||||
source = subtitle["source"]
|
||||
lang = subtitle.get("lang")
|
||||
response = requests.get(
|
||||
subtitle["url"], headers=requests_headers()
|
||||
subtitle["url"], headers=requests_headers(), timeout=30
|
||||
)
|
||||
if not response.ok:
|
||||
print(f"{self.video.youtube_id}: failed to download subtitle")
|
||||
@ -137,14 +137,18 @@ class YoutubeSubtitle:
|
||||
|
||||
return indexed
|
||||
|
||||
@staticmethod
|
||||
def _write_subtitle_file(dest_path, subtitle_str):
|
||||
def _write_subtitle_file(self, dest_path, subtitle_str):
|
||||
"""write subtitle file to disk"""
|
||||
# create folder here for first video of channel
|
||||
os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
|
||||
with open(dest_path, "w", encoding="utf-8") as subfile:
|
||||
subfile.write(subtitle_str)
|
||||
|
||||
host_uid = self.video.config["application"]["HOST_UID"]
|
||||
host_gid = self.video.config["application"]["HOST_GID"]
|
||||
if host_uid and host_gid:
|
||||
os.chown(dest_path, host_uid, host_gid)
|
||||
|
||||
@staticmethod
|
||||
def _index_subtitle(query_str):
|
||||
"""send subtitle to es for indexing"""
|
||||
|
||||
@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
|
||||
from home.src.index.channel import YoutubeChannel
|
||||
from home.src.index.filesystem import Filesystem
|
||||
from home.src.index.manual import ImportFolderScanner
|
||||
from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
|
||||
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
|
||||
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
|
||||
from home.src.ta.ta_redis import RedisArchivist
|
||||
from home.src.ta.task_manager import TaskManager
|
||||
@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
|
||||
manager.init(self)
|
||||
if not data:
|
||||
# started from scheduler
|
||||
populate = ReindexPopulate()
|
||||
print(f"[task][{self.name}] reindex outdated documents")
|
||||
self.send_progress("Add recent documents to the reindex Queue.")
|
||||
populate.add_recent()
|
||||
self.send_progress("Add outdated documents to the reindex Queue.")
|
||||
ReindexOutdated().add_outdated()
|
||||
populate.add_outdated()
|
||||
|
||||
Reindex(task=self).reindex_all()
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div id="notifications" data="channel"></div>
|
||||
<div id="notifications" data="channel reindex"></div>
|
||||
<div class="info-box info-box-2">
|
||||
<div class="info-box-item">
|
||||
<div class="round-img">
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div id="notifications" data="channel"></div>
|
||||
<div id="notifications" data="channel reindex"></div>
|
||||
<div class="info-box info-box-3">
|
||||
<div class="info-box-item">
|
||||
<div class="round-img">
|
||||
@ -131,4 +131,5 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
|
||||
{% endblock content %}
|
||||
@ -22,7 +22,7 @@
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div id="notifications" data="channel"></div>
|
||||
<div id="notifications" data="channel reindex"></div>
|
||||
<div class="view-controls">
|
||||
<div class="toggle">
|
||||
<span>Show subscribed only:</span>
|
||||
@ -66,4 +66,5 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
|
||||
{% endblock content %}
|
||||
@ -2,7 +2,7 @@
|
||||
{% load static %}
|
||||
{% block content %}
|
||||
<div class="boxed-content">
|
||||
<div id="notifications" data="setting"></div>
|
||||
<div id="notifications" data="setting reindex"></div>
|
||||
<div class="title-bar">
|
||||
<h1>User Configurations</h1>
|
||||
</div>
|
||||
|
||||
@ -31,7 +31,7 @@ function getMessages(dataOrigin) {
|
||||
function buildMessage(responseData, dataOrigin) {
|
||||
// filter relevant messages
|
||||
let messages = responseData.filter(function (value) {
|
||||
return value.group.startsWith(dataOrigin);
|
||||
return dataOrigin.split(' ').includes(value.group.split(':')[0]);
|
||||
}, dataOrigin);
|
||||
let notifications = document.getElementById('notifications');
|
||||
let currentNotifications = notifications.childElementCount;
|
||||
|
||||
@ -55,19 +55,6 @@ function createWatchStatusIndicator(videoId, videoWatchStatus) {
|
||||
return watchStatusIndicator;
|
||||
}
|
||||
|
||||
// function isWatched(youtube_id) {
|
||||
// var payload = JSON.stringify({'watched': youtube_id});
|
||||
// sendPost(payload);
|
||||
// var seenIcon = document.createElement('img');
|
||||
// seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
|
||||
// seenIcon.setAttribute('alt', 'seen-icon');
|
||||
// seenIcon.setAttribute('id', youtube_id);
|
||||
// seenIcon.setAttribute('title', "Mark as unwatched");
|
||||
// seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
|
||||
// seenIcon.classList = 'seen-icon';
|
||||
// document.getElementById(youtube_id).replaceWith(seenIcon);
|
||||
// }
|
||||
|
||||
// Removes the progress bar when passed a video id
|
||||
function removeProgressBar(videoId) {
|
||||
setProgressBar(videoId, 0, 1);
|
||||
@ -84,20 +71,6 @@ function isWatchedButton(button) {
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
// function isUnwatched(youtube_id) {
|
||||
// postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
|
||||
// var payload = JSON.stringify({'un_watched': youtube_id});
|
||||
// sendPost(payload);
|
||||
// var unseenIcon = document.createElement('img');
|
||||
// unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
|
||||
// unseenIcon.setAttribute('alt', 'unseen-icon');
|
||||
// unseenIcon.setAttribute('id', youtube_id);
|
||||
// unseenIcon.setAttribute('title', "Mark as watched");
|
||||
// unseenIcon.setAttribute('onclick', "isWatched(this.id)");
|
||||
// unseenIcon.classList = 'unseen-icon';
|
||||
// document.getElementById(youtube_id).replaceWith(unseenIcon);
|
||||
// }
|
||||
|
||||
function unsubscribe(id_unsub) {
|
||||
let payload = JSON.stringify({ unsubscribe: id_unsub });
|
||||
sendPost(payload);
|
||||
@ -163,6 +136,9 @@ function reindex(button) {
|
||||
let message = document.createElement('p');
|
||||
message.innerText = 'Reindex scheduled';
|
||||
document.getElementById('reindex-button').replaceWith(message);
|
||||
setTimeout(function () {
|
||||
checkMessages();
|
||||
}, 500);
|
||||
}
|
||||
|
||||
// download page buttons
|
||||
|
||||
Loading…
Reference in New Issue
Block a user