paginate channel migration index update

tubearchivist/config/management/commands/ta_migpath.py

@@ -98,7 +98,8 @@ class FolderMigration:
 
     def migrate_videos(self, to_migrate):
         """migrate all videos of channel"""
-        for video in to_migrate:
+        total = len(to_migrate)
+        for idx, video in enumerate(to_migrate):
             new_media_url = self._move_video_file(video)
             if not new_media_url:
                 continue
@@ -113,6 +114,9 @@ class FolderMigration:
 
             self.bulk_list.append(json.dumps(action))
             self.bulk_list.append(json.dumps(source))
+            if idx % 1000 == 0:
+                print(f"processing migration [{idx}/{total}]")
+                self.send_bulk()
 
     def _move_video_file(self, video):
         """move video file to new location"""
@@ -158,11 +162,14 @@ class FolderMigration:
             return
 
         self.bulk_list.append("\n")
+        path = "_bulk?refresh=true"
         data = "\n".join(self.bulk_list)
-        response, status = ElasticWrap("_bulk").post(data=data, ndjson=True)
+        response, status = ElasticWrap(path).post(data=data, ndjson=True)
         if not status == 200:
             print(response)
 
+        self.bulk_list = []
+
     def delete_old(self):
         """delete old empty folders"""
         all_folders = ignore_filelist(os.listdir(self.videos))