refactor thumbnail check task

tubearchivist/home/src/download/thumbnails.py

@@ -11,7 +11,7 @@ from time import sleep
 
 import requests
 from home.src.download import queue  # partial import
-from home.src.es.connect import IndexPaginate
+from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.ta.config import AppConfig
 from mutagen.mp4 import MP4, MP4Cover
 from PIL import Image, ImageFile, ImageFilter, UnidentifiedImageError
@@ -272,49 +272,59 @@ class ValidatorCallback:
 class ThumbValidator:
     """validate thumbnails"""
 
-    def download_missing(self):
+    INDEX = [
-        """download all missing artwork"""
+        {
-        self.download_missing_videos()
+            "data": {
-        self.download_missing_channels()
+                "query": {"term": {"active": {"value": True}}},
-        self.download_missing_playlists()
+                "_source": ["vid_thumb_url", "youtube_id"],
-
-    def download_missing_videos(self):
-        """get all missing video thumbnails"""
-        data = {
-            "query": {"term": {"active": {"value": True}}},
-            "sort": [{"youtube_id": {"order": "asc"}}],
-            "_source": ["vid_thumb_url", "youtube_id"],
-        }
-        paginate = IndexPaginate(
-            "ta_video", data, size=5000, callback=ValidatorCallback
-        )
-        _ = paginate.get_results()
-
-    def download_missing_channels(self):
-        """get all missing channel thumbnails"""
-        data = {
-            "query": {"term": {"channel_active": {"value": True}}},
-            "sort": [{"channel_id": {"order": "asc"}}],
-            "_source": {
-                "excludes": ["channel_description", "channel_overwrites"]
             },
-        }
+            "name": "ta_video",
-        paginate = IndexPaginate(
+        },
-            "ta_channel", data, callback=ValidatorCallback
+        {
-        )
+            "data": {
-        _ = paginate.get_results()
+                "query": {"term": {"channel_active": {"value": True}}},
+                "_source": {
+                    "excludes": ["channel_description", "channel_overwrites"]
+                },
+            },
+            "name": "ta_channel",
+        },
+        {
+            "data": {
+                "query": {"term": {"playlist_active": {"value": True}}},
+                "_source": ["playlist_id", "playlist_thumbnail"],
+            },
+            "name": "ta_playlist",
+        },
+    ]
 
-    def download_missing_playlists(self):
+    def __init__(self, task):
-        """get all missing playlist artwork"""
+        self.task = task
-        data = {
+
-            "query": {"term": {"playlist_active": {"value": True}}},
+    def validate(self):
-            "sort": [{"playlist_id": {"order": "asc"}}],
+        """validate all indexes"""
-            "_source": ["playlist_id", "playlist_thumbnail"],
+        for index in self.INDEX:
-        }
+            total = self._get_total(index["name"])
-        paginate = IndexPaginate(
+            if not total:
-            "ta_playlist", data, callback=ValidatorCallback
+                continue
-        )
+
-        _ = paginate.get_results()
+            paginate = IndexPaginate(
+                index_name=index["name"],
+                data=index["data"],
+                size=1000,
+                callback=ValidatorCallback,
+                task=self.task,
+                total=total,
+            )
+            _ = paginate.get_results()
+
+    @staticmethod
+    def _get_total(index_name):
+        """get total documents in index"""
+        path = f"{index_name}/_count"
+        response, _ = ElasticWrap(path).get()
+
+        return response.get("count")
 
 
 class ThumbFilesystem:

tubearchivist/home/tasks.py

@@ -282,10 +282,10 @@ def rescan_filesystem(self):
 
     manager.init(self)
     Filesystem(task=self).process()
-    ThumbValidator().download_missing()
+    ThumbValidator(task=self).validate()
 
 
-@shared_task(bind=True, name="thumbnail_check")
+@shared_task(bind=True, name="thumbnail_check", base=BaseTask)
 def thumbnail_check(self):
     """validate thumbnails"""
     manager = TaskManager()
@@ -294,7 +294,7 @@ def thumbnail_check(self):
         return
 
     manager.init(self)
-    ThumbValidator().download_missing()
+    ThumbValidator(task=self).validate()
 
 
 @shared_task(bind=True, name="resync_thumbs")