Fixing permissions, squashing bugs #build

Changed:
- Add recent videos to reindes queue
- Fix channel tv art fallback
- Fix notification callback notification off by one
- Fix missing reindex notification in frontend
- Change new channel folder to honoring UID/GID
- Change new subtitles to honour UID/GID
- Change manual import to honour UID/GID

tubearchivist/home/src/download/thumbnails.py

@@ -81,7 +81,7 @@ class ThumbManagerBase:
             "banner": os.path.join(
                 app_root, "static/img/default-channel-banner.jpg"
             ),
-            "art": os.path.join(
+            "tvart": os.path.join(
                 app_root, "static/img/default-channel-art.jpg"
             ),
         }
@@ -270,6 +270,7 @@ class ValidatorCallback:
             urls = (
                 channel["_source"]["channel_thumb_url"],
                 channel["_source"]["channel_banner_url"],
+                channel["_source"]["channel_tvart_url"],
             )
             handler = ThumbManager(channel["_source"]["channel_id"])
             handler.download_channel_art(urls, skip_existing=True)

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -395,8 +395,8 @@ class VideoDownloader:
         new_folder = os.path.join(videos, channel_name)
         if not os.path.exists(new_folder):
             os.makedirs(new_folder)
-            if host_uid and host_gid:
+        if host_uid and host_gid:
-                os.chown(new_folder, host_uid, host_gid)
+            os.chown(new_folder, host_uid, host_gid)
         # find real filename
         cache_dir = self.config["application"]["cache_dir"]
         all_cached = ignore_filelist(os.listdir(cache_dir + "/download/"))

tubearchivist/home/src/es/connect.py

@@ -166,7 +166,7 @@ class IndexPaginate:
     def _notify(self, processed):
         """send notification on task"""
         total = self.kwargs.get("total")
-        progress = (processed + 1) / total
+        progress = processed / total
         index_clean = self.index_name.lstrip("ta_").title()
         message = [f"Processing {index_clean}s {processed}/{total}"]
         self.kwargs.get("task").send_progress(message, progress=progress)

tubearchivist/home/src/index/manual.py

@@ -12,6 +12,7 @@ import shutil
 import subprocess
 
 from home.src.download.thumbnails import ThumbManager
+from home.src.index.comments import CommentList
 from home.src.index.video import YoutubeVideo
 from home.src.ta.config import AppConfig
 from home.src.ta.helper import ignore_filelist
@@ -141,6 +142,9 @@ class ImportFolderScanner:
 
             ManualImport(current_video, self.CONFIG).run()
 
+        video_ids = [i["video_id"] for i in self.to_import]
+        CommentList(video_ids, task=self.task).index()
+
     def _notify(self, idx, current_video):
         """send notification back to task"""
         filename = os.path.split(current_video["media"])[-1]
@@ -427,15 +431,22 @@ class ManualImport:
     def _move_to_archive(self, json_data):
         """move identified media file to archive"""
         videos = self.config["application"]["videos"]
+        host_uid = self.config["application"]["HOST_UID"]
+        host_gid = self.config["application"]["HOST_GID"]
 
         channel, file = os.path.split(json_data["media_url"])
         channel_folder = os.path.join(videos, channel)
         if not os.path.exists(channel_folder):
             os.makedirs(channel_folder)
 
+        if host_uid and host_gid:
+            os.chown(channel_folder, host_uid, host_gid)
+
         old_path = self.current_video["media"]
         new_path = os.path.join(channel_folder, file)
         shutil.move(old_path, new_path, copy_function=shutil.copyfile)
+        if host_uid and host_gid:
+            os.chown(new_path, host_uid, host_gid)
 
         base_name, _ = os.path.splitext(new_path)
         for old_path in self.current_video["subtitle"]:

tubearchivist/home/src/index/reindex.py

@@ -49,6 +49,7 @@ class ReindexBase:
     }
 
     MULTIPLY = 1.2
+    DAYS3 = 60 * 60 * 24 * 3
 
     def __init__(self):
         self.config = AppConfig().config
@@ -62,13 +63,34 @@ class ReindexBase:
         RedisQueue(queue_name=reindex_config["queue_name"]).add_list(all_ids)
 
 
-class ReindexOutdated(ReindexBase):
+class ReindexPopulate(ReindexBase):
-    """add outdated documents to reindex queue"""
+    """add outdated and recent documents to reindex queue"""
 
     def __init__(self):
         super().__init__()
         self.interval = self.config["scheduler"]["check_reindex_days"]
 
+    def add_recent(self):
+        """add recent videos to refresh"""
+        gte = datetime.fromtimestamp(self.now - self.DAYS3).date().isoformat()
+        must_list = [
+            {"term": {"active": {"value": True}}},
+            {"range": {"published": {"gte": gte}}},
+        ]
+        data = {
+            "size": 10000,
+            "query": {"bool": {"must": must_list}},
+            "sort": [{"published": {"order": "desc"}}],
+        }
+        response, _ = ElasticWrap("ta_video/_search").get(data=data)
+        hits = response["hits"]["hits"]
+        if not hits:
+            return
+
+        all_ids = [i["_source"]["youtube_id"] for i in hits]
+        reindex_config = self.REINDEX_CONFIG.get("video")
+        self.populate(all_ids, reindex_config)
+
     def add_outdated(self):
         """add outdated documents"""
         for reindex_config in self.REINDEX_CONFIG.values():

tubearchivist/home/src/index/subtitle.py

@@ -115,7 +115,7 @@ class YoutubeSubtitle:
             source = subtitle["source"]
             lang = subtitle.get("lang")
             response = requests.get(
-                subtitle["url"], headers=requests_headers()
+                subtitle["url"], headers=requests_headers(), timeout=30
             )
             if not response.ok:
                 print(f"{self.video.youtube_id}: failed to download subtitle")
@@ -137,14 +137,18 @@ class YoutubeSubtitle:
 
         return indexed
 
-    @staticmethod
+    def _write_subtitle_file(self, dest_path, subtitle_str):
-    def _write_subtitle_file(dest_path, subtitle_str):
         """write subtitle file to disk"""
         # create folder here for first video of channel
         os.makedirs(os.path.split(dest_path)[0], exist_ok=True)
         with open(dest_path, "w", encoding="utf-8") as subfile:
             subfile.write(subtitle_str)
 
+        host_uid = self.video.config["application"]["HOST_UID"]
+        host_gid = self.video.config["application"]["HOST_GID"]
+        if host_uid and host_gid:
+            os.chown(dest_path, host_uid, host_gid)
+
     @staticmethod
     def _index_subtitle(query_str):
         """send subtitle to es for indexing"""

tubearchivist/home/tasks.py

@@ -21,7 +21,7 @@ from home.src.es.index_setup import ElasitIndexWrap
 from home.src.index.channel import YoutubeChannel
 from home.src.index.filesystem import Filesystem
 from home.src.index.manual import ImportFolderScanner
-from home.src.index.reindex import Reindex, ReindexManual, ReindexOutdated
+from home.src.index.reindex import Reindex, ReindexManual, ReindexPopulate
 from home.src.ta.config import AppConfig, ReleaseVersion, ScheduleBuilder
 from home.src.ta.ta_redis import RedisArchivist
 from home.src.ta.task_manager import TaskManager
@@ -220,9 +220,12 @@ def check_reindex(self, data=False, extract_videos=False):
     manager.init(self)
     if not data:
         # started from scheduler
+        populate = ReindexPopulate()
         print(f"[task][{self.name}] reindex outdated documents")
+        self.send_progress("Add recent documents to the reindex Queue.")
+        populate.add_recent()
         self.send_progress("Add outdated documents to the reindex Queue.")
-        ReindexOutdated().add_outdated()
+        populate.add_outdated()
 
     Reindex(task=self).reindex_all()
 

tubearchivist/home/templates/home/channel_id.html

@@ -22,7 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
-    <div id="notifications" data="channel"></div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="info-box info-box-2">
         <div class="info-box-item">
             <div class="round-img">

tubearchivist/home/templates/home/channel_id_about.html

@@ -22,7 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
-    <div id="notifications" data="channel"></div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="info-box info-box-3">
         <div class="info-box-item">
             <div class="round-img">
@@ -131,4 +131,5 @@
         </div>
     </div>
 </div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
 {% endblock content %}

tubearchivist/home/templates/home/channel_id_playlist.html

@@ -22,7 +22,7 @@
             <a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
         {% endif %}
     </div>
-    <div id="notifications" data="channel"></div>
+    <div id="notifications" data="channel reindex"></div>
     <div class="view-controls">
         <div class="toggle">
             <span>Show subscribed only:</span>
@@ -66,4 +66,5 @@
         {% endif %}
     </div>
 </div>
+<script type="text/javascript" src="{% static 'progress.js' %}"></script>
 {% endblock content %}

tubearchivist/home/templates/home/settings.html

@@ -2,7 +2,7 @@
 {% load static %}
 {% block content %}
 <div class="boxed-content">
-    <div id="notifications" data="setting"></div>
+    <div id="notifications" data="setting reindex"></div>
     <div class="title-bar">
         <h1>User Configurations</h1>
     </div>

tubearchivist/static/progress.js

@@ -31,7 +31,7 @@ function getMessages(dataOrigin) {
 function buildMessage(responseData, dataOrigin) {
   // filter relevant messages
   let messages = responseData.filter(function (value) {
-    return value.group.startsWith(dataOrigin);
+    return dataOrigin.split(' ').includes(value.group.split(':')[0]);
   }, dataOrigin);
   let notifications = document.getElementById('notifications');
   let currentNotifications = notifications.childElementCount;

tubearchivist/static/script.js

@@ -55,19 +55,6 @@ function createWatchStatusIndicator(videoId, videoWatchStatus) {
   return watchStatusIndicator;
 }
 
-// function isWatched(youtube_id) {
-//     var payload = JSON.stringify({'watched': youtube_id});
-//     sendPost(payload);
-//     var seenIcon = document.createElement('img');
-//     seenIcon.setAttribute('src', "/static/img/icon-seen.svg");
-//     seenIcon.setAttribute('alt', 'seen-icon');
-//     seenIcon.setAttribute('id', youtube_id);
-//     seenIcon.setAttribute('title', "Mark as unwatched");
-//     seenIcon.setAttribute('onclick', "isUnwatched(this.id)");
-//     seenIcon.classList = 'seen-icon';
-//     document.getElementById(youtube_id).replaceWith(seenIcon);
-// }
-
 // Removes the progress bar when passed a video id
 function removeProgressBar(videoId) {
   setProgressBar(videoId, 0, 1);
@@ -84,20 +71,6 @@ function isWatchedButton(button) {
   }, 1000);
 }
 
-// function isUnwatched(youtube_id) {
-//     postVideoProgress(youtube_id, 0); // Reset video progress on unwatched;
-//     var payload = JSON.stringify({'un_watched': youtube_id});
-//     sendPost(payload);
-//     var unseenIcon = document.createElement('img');
-//     unseenIcon.setAttribute('src', "/static/img/icon-unseen.svg");
-//     unseenIcon.setAttribute('alt', 'unseen-icon');
-//     unseenIcon.setAttribute('id', youtube_id);
-//     unseenIcon.setAttribute('title', "Mark as watched");
-//     unseenIcon.setAttribute('onclick', "isWatched(this.id)");
-//     unseenIcon.classList = 'unseen-icon';
-//     document.getElementById(youtube_id).replaceWith(unseenIcon);
-// }
-
 function unsubscribe(id_unsub) {
   let payload = JSON.stringify({ unsubscribe: id_unsub });
   sendPost(payload);
@@ -163,6 +136,9 @@ function reindex(button) {
   let message = document.createElement('p');
   message.innerText = 'Reindex scheduled';
   document.getElementById('reindex-button').replaceWith(message);
+  setTimeout(function () {
+    checkMessages();
+  }, 500);
 }
 
 // download page buttons