add notification for comment indexing

This commit is contained in:
simon 2022-11-22 11:23:34 +07:00
parent b48b7c7e72
commit 90258e75e6
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
2 changed files with 33 additions and 4 deletions

View File

@ -146,11 +146,22 @@ class DownloadPostProcess:
if not self.download.config["downloads"]["comment_max"]:
return
for video_id in self.download.videos:
total_videos = len(self.download.videos)
for idx, video_id in enumerate(self.download.videos):
comment = Comments(video_id, config=self.download.config)
comment.build_json()
comment.build_json(notify=(idx, total_videos))
comment.upload_comments()
key = "message:download"
message = {
"status": key,
"level": "info",
"title": "Download and index comments finished",
"message": f"added comments for {total_videos} videos",
}
RedisArchivist().set_message(key, message, expire=4)
class VideoDownloader:
"""

View File

@ -10,6 +10,7 @@ from datetime import datetime
from home.src.download.yt_dlp_base import YtWrap
from home.src.es.connect import ElasticWrap
from home.src.ta.config import AppConfig
from home.src.ta.ta_redis import RedisArchivist
class Comments:
@ -23,14 +24,14 @@ class Comments:
self.is_activated = False
self.comments_format = False
def build_json(self):
def build_json(self, notify=False):
"""build json document for es"""
print(f"{self.youtube_id}: get comments")
self.check_config()
if not self.is_activated:
return
self._send_notification(notify)
comments_raw, channel_id = self.get_yt_comments()
self.format_comments(comments_raw)
@ -48,6 +49,23 @@ class Comments:
self.is_activated = bool(self.config["downloads"]["comment_max"])
@staticmethod
def _send_notification(notify):
"""send notification for download post process message"""
if not notify:
return
key = "message:download"
idx, total_videos = notify
message = {
"status": key,
"level": "info",
"title": "Download and index comments",
"message": f"Progress: {idx + 1}/{total_videos}",
}
RedisArchivist().set_message(key, message)
def build_yt_obs(self):
"""
get extractor config