index comments as a DownloadPostProcess

tubearchivist/home/src/download/yt_dlp_handler.py

@@ -15,6 +15,7 @@ from home.src.download.subscriptions import PlaylistSubscription
 from home.src.download.yt_dlp_base import CookieHandler, YtWrap
 from home.src.es.connect import ElasticWrap, IndexPaginate
 from home.src.index.channel import YoutubeChannel
+from home.src.index.comments import Comments
 from home.src.index.playlist import YoutubePlaylist
 from home.src.index.video import YoutubeVideo, index_new_video
 from home.src.ta.config import AppConfig
@@ -39,6 +40,7 @@ class DownloadPostProcess:
         self.auto_delete_all()
         self.auto_delete_overwrites()
         self.validate_playlists()
+        self.get_comments()
 
     def auto_delete_all(self):
         """handle auto delete"""
@@ -139,6 +141,16 @@ class DownloadPostProcess:
 
         RedisArchivist().set_message(key, mess_dict, expire=expire)
 
+    def get_comments(self):
+        """get comments from youtube"""
+        if not self.download.config["downloads"]["comment_max"]:
+            return
+
+        for video_id in self.download.videos:
+            comment = Comments(video_id)
+            comment.build_json()
+            comment.upload_comments()
+
 
 class VideoDownloader:
     """
@@ -155,6 +167,7 @@ class VideoDownloader:
         self.config = AppConfig().config
         self._build_obs()
         self.channels = set()
+        self.videos = set()
 
     def run_queue(self):
         """setup download queue in redis loop until no more items"""
@@ -187,6 +200,7 @@ class VideoDownloader:
                 youtube_id, video_overwrites=self.video_overwrites
             )
             self.channels.add(vid_dict["channel"]["channel_id"])
+            self.videos.add(vid_dict["youtube_id"])
             mess_dict = {
                 "status": self.MSG,
                 "level": "info",

tubearchivist/home/src/index/comments.py

@@ -23,6 +23,7 @@ class Comments:
 
     def build_json(self):
         """build json document for es"""
+        print(f"{self.youtube_id}: get comments")
         self._check_config()
         comments_raw = self.get_yt_comments()
         comments_format = self.format_comments(comments_raw)
@@ -63,7 +64,6 @@ class Comments:
 
     def get_yt_comments(self):
         """get comments from youtube"""
-        print("comments: get comments")
         yt_obs = self.build_yt_obs()
         info_json = YtWrap(yt_obs).extract(self.youtube_id)
         comments_raw = info_json.get("comments")