mirror of
https://github.com/tubearchivist/tubearchivist.git
synced 2024-09-07 17:58:47 +00:00
configure comments extraction
This commit is contained in:
parent
11c122cba3
commit
172ced7129
|
@ -27,6 +27,8 @@
|
||||||
"subtitle": false,
|
"subtitle": false,
|
||||||
"subtitle_source": false,
|
"subtitle_source": false,
|
||||||
"subtitle_index": false,
|
"subtitle_index": false,
|
||||||
|
"comment_max": false,
|
||||||
|
"comment_sort": "top",
|
||||||
"cookie_import": false,
|
"cookie_import": false,
|
||||||
"throttledratelimit": false,
|
"throttledratelimit": false,
|
||||||
"integrate_ryd": false,
|
"integrate_ryd": false,
|
||||||
|
|
|
@ -92,6 +92,12 @@ class ApplicationSettingsForm(forms.Form):
|
||||||
("1", "enable subtitle index"),
|
("1", "enable subtitle index"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
COMMENT_SORT_CHOICES = [
|
||||||
|
("", "-- change comments sort settings --"),
|
||||||
|
("top", "sort comments by top"),
|
||||||
|
("new", "sort comments by new"),
|
||||||
|
]
|
||||||
|
|
||||||
COOKIE_IMPORT_CHOICES = [
|
COOKIE_IMPORT_CHOICES = [
|
||||||
("", "-- change cookie settings"),
|
("", "-- change cookie settings"),
|
||||||
("0", "disable cookie"),
|
("0", "disable cookie"),
|
||||||
|
@ -120,6 +126,10 @@ class ApplicationSettingsForm(forms.Form):
|
||||||
downloads_subtitle_index = forms.ChoiceField(
|
downloads_subtitle_index = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
|
widget=forms.Select, choices=SUBTITLE_INDEX_CHOICES, required=False
|
||||||
)
|
)
|
||||||
|
downloads_comment_max = forms.CharField(required=False)
|
||||||
|
downloads_comment_sort = forms.ChoiceField(
|
||||||
|
widget=forms.Select, choices=COMMENT_SORT_CHOICES, required=False
|
||||||
|
)
|
||||||
downloads_cookie_import = forms.ChoiceField(
|
downloads_cookie_import = forms.ChoiceField(
|
||||||
widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
|
widget=forms.Select, choices=COOKIE_IMPORT_CHOICES, required=False
|
||||||
)
|
)
|
||||||
|
|
|
@ -9,19 +9,21 @@ from datetime import datetime
|
||||||
|
|
||||||
from home.src.download.yt_dlp_base import YtWrap
|
from home.src.download.yt_dlp_base import YtWrap
|
||||||
from home.src.es.connect import ElasticWrap
|
from home.src.es.connect import ElasticWrap
|
||||||
|
from home.src.ta.config import AppConfig
|
||||||
|
|
||||||
|
|
||||||
class Comments:
|
class Comments:
|
||||||
"""hold all comments functionality"""
|
"""hold all comments functionality"""
|
||||||
|
|
||||||
def __init__(self, youtube_id):
|
def __init__(self, youtube_id, config=False):
|
||||||
self.youtube_id = youtube_id
|
self.youtube_id = youtube_id
|
||||||
self.es_path = f"ta_comment/_doc/{youtube_id}"
|
self.es_path = f"ta_comment/_doc/{youtube_id}"
|
||||||
self.max_comments = "all,100,all,30"
|
|
||||||
self.json_data = False
|
self.json_data = False
|
||||||
|
self.config = config
|
||||||
|
|
||||||
def build_json(self):
|
def build_json(self):
|
||||||
"""build json document for es"""
|
"""build json document for es"""
|
||||||
|
self._check_config()
|
||||||
comments_raw = self.get_yt_comments()
|
comments_raw = self.get_yt_comments()
|
||||||
comments_format = self.format_comments(comments_raw)
|
comments_format = self.format_comments(comments_raw)
|
||||||
|
|
||||||
|
@ -31,13 +33,19 @@ class Comments:
|
||||||
"comment_comments": comments_format,
|
"comment_comments": comments_format,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _check_config(self):
|
||||||
|
"""read config if not attached"""
|
||||||
|
if not self.config:
|
||||||
|
self.config = AppConfig().config
|
||||||
|
|
||||||
def build_yt_obs(self):
|
def build_yt_obs(self):
|
||||||
"""
|
"""
|
||||||
get extractor config
|
get extractor config
|
||||||
max-comments,max-parents,max-replies,max-replies-per-thread
|
max-comments,max-parents,max-replies,max-replies-per-thread
|
||||||
"""
|
"""
|
||||||
max_comments_list = [i.strip() for i in self.max_comments.split(",")]
|
max_comments = self.config["downloads"]["comment_max"]
|
||||||
comment_sort = "top"
|
max_comments_list = [i.strip() for i in max_comments.split(",")]
|
||||||
|
comment_sort = self.config["downloads"]["comment_sort"]
|
||||||
|
|
||||||
yt_obs = {
|
yt_obs = {
|
||||||
"skip_download": True,
|
"skip_download": True,
|
||||||
|
@ -55,7 +63,7 @@ class Comments:
|
||||||
|
|
||||||
def get_yt_comments(self):
|
def get_yt_comments(self):
|
||||||
"""get comments from youtube"""
|
"""get comments from youtube"""
|
||||||
print(f"comments: get comments with format {self.max_comments}")
|
print("comments: get comments")
|
||||||
yt_obs = self.build_yt_obs()
|
yt_obs = self.build_yt_obs()
|
||||||
info_json = YtWrap(yt_obs).extract(self.youtube_id)
|
info_json = YtWrap(yt_obs).extract(self.youtube_id)
|
||||||
comments_raw = info_json.get("comments")
|
comments_raw = info_json.get("comments")
|
||||||
|
|
|
@ -114,6 +114,24 @@
|
||||||
{{ app_form.downloads_subtitle_index }}
|
{{ app_form.downloads_subtitle_index }}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="settings-group">
|
||||||
|
<h2 id="comments">Comments</h2>
|
||||||
|
<div class="settings-item">
|
||||||
|
<p>Download and index comments: <span class="settings-current">{{ config.downloads.comment_max }}</span><br>
|
||||||
|
<i>Follow the yt-dlp max_comments documentation, <a href="https://github.com/yt-dlp/yt-dlp#youtube" target="_blank">max-comments,max-parents,max-replies,max-replies-per-thread</a>:</i><br>
|
||||||
|
<p>Example configurations:</p>
|
||||||
|
<ul>
|
||||||
|
<li><span class="settings-current">all,100,all,30</span>: Get 100 max-parents and 30 max-replies-per-thread.</li>
|
||||||
|
<li><span class="settings-current">1000,all,all,50</span>: Get a total of 1000 comments over all, 50 replies per thread.</li>
|
||||||
|
</ul>
|
||||||
|
{{ app_form.downloads_comment_max }}</p>
|
||||||
|
</div>
|
||||||
|
<div class="settings-item">
|
||||||
|
<p>Selected comment sort method: <span class="settings-current">{{ config.downloads.comment_sort }}</span><br>
|
||||||
|
<i>Select how many comments and threads to download:</i><br>
|
||||||
|
{{ app_form.downloads_comment_sort }}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="settings-group">
|
<div class="settings-group">
|
||||||
<h2 id="format">Cookie</h2>
|
<h2 id="format">Cookie</h2>
|
||||||
<div class="settings-item">
|
<div class="settings-item">
|
||||||
|
|
Loading…
Reference in New Issue
Block a user