fix dl error retry logic, store and return error, #477

tubearchivist/home/src/download/yt_dlp_base.py

@@ -48,11 +48,11 @@ class YtWrap:
         with yt_dlp.YoutubeDL(self.obs) as ydl:
             try:
                 ydl.download([url])
-            except yt_dlp.utils.DownloadError:
+            except yt_dlp.utils.DownloadError as err:
                 print(f"{url}: failed to download.")
-                return False
+                return False, str(err)
 
-        return True
+        return True, True
 
     def extract(self, url):
         """make extract request"""

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -203,12 +203,13 @@ class VideoDownloader:
     def _get_next(self, auto_only):
         """get next item in queue"""
         must_list = [{"term": {"status": {"value": "pending"}}}]
+        must_not_list = [{"exists": {"field": "message"}}]
         if auto_only:
             must_list.append({"term": {"auto_start": {"value": True}}})
 
         data = {
             "size": 1,
-            "query": {"bool": {"must": must_list}},
+            "query": {"bool": {"must": must_list, "must_not": must_not_list}},
             "sort": [
                 {"auto_start": {"order": "desc"}},
                 {"timestamp": {"order": "asc"}},
@@ -344,7 +345,9 @@ class VideoDownloader:
             if youtube_id in file_name:
                 obs["outtmpl"] = os.path.join(dl_cache, file_name)
 
-        success = YtWrap(obs, self.config).download(youtube_id)
+        success, message = YtWrap(obs, self.config).download(youtube_id)
+        if not success:
+            self._handle_error(youtube_id, message)
 
         if self.obs["writethumbnail"]:
             # webp files don't get cleaned up automatically
@@ -356,6 +359,12 @@ class VideoDownloader:
 
         return success
 
+    @staticmethod
+    def _handle_error(youtube_id, message):
+        """store error message"""
+        data = {"doc": {"message": message}}
+        _, _ = ElasticWrap(f"ta_download/_update/{youtube_id}").post(data=data)
+
     def move_to_archive(self, vid_dict):
         """move downloaded video from cache to archive"""
         videos = self.config["application"]["videos"]

tubearchivist/home/src/es/index_mapping.json

@@ -380,6 +380,9 @@
                 },
                 "auto_start": {
                     "type": "boolean"
+                },
+                "message": {
+                    "type": "text"
                 }
             },
             "expected_set": {

tubearchivist/home/templates/home/downloads.html

@@ -97,6 +97,9 @@
                             <a href="https://www.youtube.com/watch?v={{ video.source.youtube_id }}" target="_blank"><h3>{{ video.source.title }}</h3></a>
                         </div>
                         <p>Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}</p>
+                        {% if video.source.message %}
+                            <p class="danger-zone">{{ video.source.message }}</p>
+                        {% endif %}
                         <div>
                             {% if show_ignored_only %}
                                 <button data-id="{{ video.source.youtube_id }}" onclick="forgetIgnore(this)">Forget</button>