handling genering index write errors, #91

README.md

@@ -107,6 +107,9 @@ chown 1000:0 /path/to/mount/point
 ```
 This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
 
+### Disk usage
+The Elasticsearch index will turn to *read only* if the disk usage of the container goes above 95% until the usage drops below 90% again. Similar to that, TubeArchivist will become all sorts of messed up when running out of disk space. There are some error messages in the logs when that happens, but it's best to make sure to have enough disk space before starting to download.
+
 ## Getting Started
 1. Go through the **settings** page and look at the available options. Particularly set *Download Format* to your desired video quality before downloading. **Tube Archivist** downloads the best available quality by default. To support iOS or MacOS a compatible format must be specified. For example:
 ```

tubearchivist/home/src/download.py

@@ -92,6 +92,7 @@ class PendingList:
         )
         if not request.ok:
             print(request)
+            raise ValueError("failed to add video to download queue")
 
         return all_videos_added
 
@@ -267,6 +268,7 @@ class PendingList:
         RedisArchivist().set_message("progress:download", mess_dict)
         if not request.ok:
             print(request)
+            raise ValueError("failed to set video to ignore")
 
 
 class ChannelSubscription:
@@ -361,6 +363,7 @@ class ChannelSubscription:
         )
         if not request.ok:
             print(request.text)
+            raise ValueError("failed change subscribe status")
         # sync to videos
         channel_handler.sync_to_videos()
         if channel_handler.source == "scraped":
@@ -445,6 +448,7 @@ class PlaylistSubscription:
         )
         if not response.ok:
             print(response.text)
+            raise ValueError("failed to change subscribe status")
 
         return True
 

tubearchivist/home/src/index.py

@@ -171,6 +171,7 @@ class YoutubeChannel:
         print(f"added {self.channel_id} to es")
         if not response.ok:
             print(response.text)
+            raise ValueError("failed to add channel to index")
 
     def sync_to_videos(self):
         """sync new channel_dict to all videos of channel"""
@@ -408,11 +409,12 @@ class YoutubeVideo:
         return es_vid_dict
 
     def upload_to_es(self):
-        """upload channel data to elastic search"""
+        """upload video data to elastic search"""
         url = f"{self.ES_URL}/ta_video/_doc/{self.youtube_id}/?refresh=true"
         response = requests.put(url, json=self.vid_dict, auth=self.ES_AUTH)
         if not response.ok:
             print(response.text)
+            raise ValueError("failed to add video to index")
 
     def deactivate(self):
         """deactivate document on extractor error"""
@@ -552,6 +554,7 @@ class YoutubePlaylist:
         response = requests.put(url, json=playlist, auth=self.ES_AUTH)
         if not response.ok:
             print(response.text)
+            raise ValueError("failed to add playlist to index")
 
     def add_vids_to_playlist(self):
         """sync the playlist id to videos"""
@@ -738,6 +741,7 @@ class WatchState:
         )
         if not request.ok:
             print(request.text)
+            raise ValueError("failed to mark video as watched")
 
     def mark_channel_watched(self):
         """change watched status of every video in channel"""
@@ -768,6 +772,7 @@ class WatchState:
         )
         if not request.ok:
             print(request.text)
+            raise ValueError("failed mark channel as watched")
 
     def mark_playlist_watched(self):
         """change watched state of all videos in playlist"""
@@ -796,6 +801,7 @@ class WatchState:
         )
         if not request.ok:
             print(request.text)
+            raise ValueError("failed mark playlist as watched")
 
 
 class IndexPaginate: