tubearchivist/tubearchivist
1
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist.git synced 2024-12-23 10:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Fixing permissions, squashing bugs #build

Browse Source 
Changed:
- Add recent videos to reindes queue
- Fix channel tv art fallback
- Fix notification callback notification off by one
- Fix missing reindex notification in frontend
- Change new channel folder to honoring UID/GID
- Change new subtitles to honour UID/GID
- Change manual import to honour UID/GID
This commit is contained in:
simon 2023-04-08 17:46:41 +07:00
commit 5e841bf7f8
13 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tubearchivist/home/src/download/thumbnails.py
View File

@ -81,7 +81,7 @@ class ThumbManagerBase:
"banner": os.path.join(
app_root, "static/img/default-channel-banner.jpg"
),
"art": os.path.join(
"tvart": os.path.join(
app_root, "static/img/default-channel-art.jpg"
),
}
@ -270,6 +270,7 @@ class ValidatorCallback:
urls = (
channel["_source"]["channel_thumb_url"],
channel["_source"]["channel_banner_url"],
channel["_source"]["channel_tvart_url"],
)
handler = ThumbManager(channel["_source"]["channel_id"])
handler.download_channel_art(urls, skip_existing=True)

2
tubearchivist/home/src/es/connect.py
View File

@ -166,7 +166,7 @@ class IndexPaginate:
def _notify(self, processed):
"""send notification on task"""
total = self.kwargs.get("total")
progress = (processed + 1) / total
progress = processed / total
index_clean = self.index_name.lstrip("ta_").title()
message = [f"Processing {index_clean}s {processed}/{total}"]
self.kwargs.get("task").send_progress(message, progress=progress)

11
tubearchivist/home/src/index/manual.py
View File

@ -12,6 +12,7 @@ import shutil
import subprocess
from home.src.download.thumbnails import ThumbManager
from home.src.index.comments import CommentList
from home.src.index.video import YoutubeVideo
from home.src.ta.config import AppConfig
from home.src.ta.helper import ignore_filelist
@ -141,6 +142,9 @@ class ImportFolderScanner:
ManualImport(current_video, self.CONFIG).run()
video_ids = [i["video_id"] for i in self.to_import]
CommentList(video_ids, task=self.task).index()
def _notify(self, idx, current_video):
"""send notification back to task"""
filename = os.path.split(current_video["media"])[-1]
@ -427,15 +431,22 @@ class ManualImport:
def _move_to_archive(self, json_data):
"""move identified media file to archive"""
videos = self.config["application"]["videos"]
host_uid = self.config["application"]["HOST_UID"]
host_gid = self.config["application"]["HOST_GID"]
channel, file = os.path.split(json_data["media_url"])
channel_folder = os.path.join(videos, channel)
if not os.path.exists(channel_folder):
os.makedirs(channel_folder)
if host_uid and host_gid:
os.chown(channel_folder, host_uid, host_gid)
old_path = self.current_video["media"]
new_path = os.path.join(channel_folder, file)
shutil.move(old_path, new_path, copy_function=shutil.copyfile)
if host_uid and host_gid:
os.chown(new_path, host_uid, host_gid)
base_name, _ = os.path.splitext(new_path)
for old_path in self.current_video["subtitle"]:

26
tubearchivist/home/src/index/reindex.py
View File

@ -49,6 +49,7 @@ class ReindexBase:
}
MULTIPLY = 1.2
DAYS3 = 60 * 60 * 24 * 3
def __init__(self):
self.config = AppConfig().config
@ -62,13 +63,34 @@ class ReindexBase:
RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
class ReindexOutdated(ReindexBase):
"""add outdated documents to reindex queue"""
class ReindexPopulate(ReindexBase):
"""add outdated and recent documents to reindex queue"""
def __init__(self):
super().__init__()
self.interval = self.config["scheduler"]["check_reindex_days"]
def add_recent(self):
"""add recent videos to refresh"""
gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
must_list = [
{"term": {"active": {"value": True}}},
{"range": {"published": {"gte": gte}}},
]
data = {
"size": 10000,
"query": {"bool": {"must": must_list}},
"sort": [{"published": {"order": "desc"}}],
}
response, _ = ElasticWrap("ta_video/_search").get(data=data)
hits = response["hits"]["hits"]
if not hits:
return
all_ids = [i["_source"]["youtube_id"] for i in hits]
reindex_config = self.REINDEX_CONFIG.get("video")
self.populate(all_ids, reindex_config)
def add_outdated(self):
"""add outdated documents"""
for reindex_config in self.REINDEX_CONFIG.values():

10
tubearchivist/home/src/index/subtitle.py
View File

@ -115,7 +115,7 @@ class YoutubeSubtitle:
source = subtitle["source"]
lang = subtitle.get("lang")
response = requests.get(
subtitle["url"], headers=requests_headers()
subtitle["url"], headers=requests_headers(), timeout=30
)
if not response.ok:
print(f"{self.video.youtube_id}: failed to download subtitle")
@ -137,14 +137,18 @@ class YoutubeSubtitle:
return indexed
@staticmethod
def _write_subtitle_file(dest_path, subtitle_str):
def _write_subtitle_file(self, dest_path, subtitle_str):
"""write subtitle file to disk"""
# create folder here for first video of channel
os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
with open(dest_path, "w", encoding="utf-8") as subfile:
subfile.write(subtitle_str)
host_uid = self.video.config["application"]["HOST_UID"]
host_gid = self.video.config["application"]["HOST_GID"]
if host_uid and host_gid:
os.chown(dest_path, host_uid, host_gid)
@staticmethod
def _index_subtitle(query_str):
"""send subtitle to es for indexing"""

7
tubearchivist/home/tasks.py
View File

@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
from home.src.index.channel import YoutubeChannel
from home.src.index.filesystem import Filesystem
from home.src.index.manual import ImportFolderScanner
from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
from home.src.ta.ta_redis import RedisArchivist
from home.src.ta.task_manager import TaskManager
@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
manager.init(self)
if not data:
# started from scheduler
populate = ReindexPopulate()
print(f"[task][{self.name}] reindex outdated documents")
self.send_progress("Add recent documents to the reindex Queue.")
populate.add_recent()
self.send_progress("Add outdated documents to the reindex Queue.")
ReindexOutdated().add_outdated()
populate.add_outdated()
Reindex(task=self).reindex_all()

2
tubearchivist/home/templates/home/channel_id.html
View File

@ -22,7 +22,7 @@
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
</div>
<div id="notifications" data="channel"></div>
<div id="notifications" data="channel reindex"></div>
<div class="info-box info-box-2">
<div class="info-box-item">
<div class="round-img">

3
tubearchivist/home/templates/home/channel_id_about.html
View File

@ -22,7 +22,7 @@
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
</div>
<div id="notifications" data="channel"></div>
<div id="notifications" data="channel reindex"></div>
<div class="info-box info-box-3">
<div class="info-box-item">
<div class="round-img">
@ -131,4 +131,5 @@
</div>
</div>
</div>
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
{% endblock content %}

3
tubearchivist/home/templates/home/channel_id_playlist.html
View File

@ -22,7 +22,7 @@
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
{% endif %}
</div>
<div id="notifications" data="channel"></div>
<div id="notifications" data="channel reindex"></div>
<div class="view-controls">
<div class="toggle">
<span>Show subscribed only:</span>
@ -66,4 +66,5 @@
{% endif %}
</div>
</div>
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
{% endblock content %}

2
tubearchivist/home/templates/home/settings.html
View File

@ -2,7 +2,7 @@
{% load static %}
{% block content %}
<div class="boxed-content">
<div id="notifications" data="setting"></div>
<div id="notifications" data="setting reindex"></div>
<div class="title-bar">
<h1>User Configurations</h1>
</div>

2
tubearchivist/static/progress.js
View File

@ -31,7 +31,7 @@ function getMessages(dataOrigin) {
function buildMessage(responseData, dataOrigin) {
// filter relevant messages
let messages = responseData.filter(function (value) {
return value.group.startsWith(dataOrigin);
return dataOrigin.split(' ').includes(value.group.split(':')[0]);
}, dataOrigin);
let notifications = document.getElementById('notifications');
let currentNotifications = notifications.childElementCount;

30
tubearchivist/static/script.js
View File

@ -55,19 +55,6 @@ function createWatchStatusIndicator(videoId, videoWatchStatus) {
return watchStatusIndicator;
}
// function isWatched(youtube_id) {
// var payload = JSON.stringify({'watched': youtube_id});
// sendPost(payload);
// var seenIcon = document.createElement('img');
// seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
// seenIcon.setAttribute('alt', 'seen-icon');
// seenIcon.setAttribute('id', youtube_id);
// seenIcon.setAttribute('title', "Mark as unwatched");
// seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
// seenIcon.classList = 'seen-icon';
// document.getElementById(youtube_id).replaceWith(seenIcon);
// }
// Removes the progress bar when passed a video id
function removeProgressBar(videoId) {
setProgressBar(videoId, 0, 1);
@ -84,20 +71,6 @@ function isWatchedButton(button) {
}, 1000);
}
// function isUnwatched(youtube_id) {
// postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
// var payload = JSON.stringify({'un_watched': youtube_id});
// sendPost(payload);
// var unseenIcon = document.createElement('img');
// unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
// unseenIcon.setAttribute('alt', 'unseen-icon');
// unseenIcon.setAttribute('id', youtube_id);
// unseenIcon.setAttribute('title', "Mark as watched");
// unseenIcon.setAttribute('onclick', "isWatched(this.id)");
// unseenIcon.classList = 'unseen-icon';
// document.getElementById(youtube_id).replaceWith(unseenIcon);
// }
function unsubscribe(id_unsub) {
let payload = JSON.stringify({ unsubscribe: id_unsub });
sendPost(payload);
@ -163,6 +136,9 @@ function reindex(button) {
let message = document.createElement('p');
message.innerText = 'Reindex scheduled';
document.getElementById('reindex-button').replaceWith(message);
setTimeout(function () {
checkMessages();
}, 500);
}
// download page buttons