appending reindex task to rescan subscriptions

tubearchivist/home/src/reindex.py

@@ -20,21 +20,27 @@ from home.src.index import (
     YoutubeVideo,
     index_new_video
 )
-from home.src.helper import get_total_hits, clean_string, set_message
+from home.src.helper import (
+    get_total_hits,
+    clean_string,
+    set_message,
+    get_message
+)
 
 
 class Reindex:
     """ check for outdated documents and refresh data from youtube """
 
     def __init__(self):
-        self.video_daily, self.channel_daily = self.get_daily()
-        self.all_youtube_ids = False
-        self.all_channel_ids = False
         # config
         config = AppConfig().config
         self.sleep_interval = config['downloads']['sleep_interval']
         self.es_url = config['application']['es_url']
         self.refresh_interval = 90
+        # scan
+        self.video_daily, self.channel_daily = self.get_daily()
+        self.all_youtube_ids = False
+        self.all_channel_ids = False
 
     def get_daily(self):
         """ get daily refresh values """
@@ -178,11 +184,13 @@ class Reindex:
     def reindex(self):
         """ reindex what's needed """
         # videos
+        print(f'reindexing {len(self.all_youtube_ids)} videos')
         for youtube_id in self.all_youtube_ids:
             self.reindex_single_video(youtube_id)
             if self.sleep_interval:
                 sleep(self.sleep_interval)
         # channels
+        print(f'reindexing {len(self.all_channel_ids)} channels')
         for channel_id in self.all_channel_ids:
             self.reindex_single_channel(channel_id)
             if self.sleep_interval:
@@ -342,6 +350,14 @@ def scan_filesystem():
 
 def reindex_old_documents():
     """ daily refresh of old documents """
+    # check needed last run
+    now = int(datetime.now().strftime("%s"))
+    last_reindex = get_message('last_reindex')
+    if isinstance(last_reindex, int) and now - last_reindex < 60 * 60 * 12:
+        return
+    # continue if needed
     reindex_handler = Reindex()
     reindex_handler.check_outdated()
     reindex_handler.reindex()
+    # set timestamp
+    set_message('last_reindex', now, expire=False)

tubearchivist/home/tasks.py

@@ -14,6 +14,7 @@ from home.src.download import (
     VideoDownloader
 )
 from home.src.config import AppConfig
+from home.src.reindex import reindex_old_documents
 
 
 CONFIG = AppConfig().config
@@ -33,6 +34,8 @@ def update_subscribed():
     if missing_videos:
         pending_handler = PendingList()
         pending_handler.add_to_pending(missing_videos)
+    # check if reindex is needed
+    check_reindex.delay()
 
 
 @shared_task
@@ -61,3 +64,9 @@ def extrac_dl(youtube_ids):
     pending_handler = PendingList()
     missing_videos = pending_handler.parse_url_list(youtube_ids)
     pending_handler.add_to_pending(missing_videos)
+
+
+@shared_task
+def check_reindex():
+    """ run the reindex main command """
+    reindex_old_documents()