From c85be4584608f9a2ede2d95ad9ad5281f84ca047 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 5 Jan 2022 14:43:24 +0700 Subject: [PATCH] extend ryd to refresh when missing average_rating --- tubearchivist/home/src/reindex.py | 24 +++++++++++++++++++ .../home/templates/home/settings.html | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index 6b9ee33..643ab7c 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -42,6 +42,7 @@ class Reindex: self.es_url = config["application"]["es_url"] self.es_auth = config["application"]["es_auth"] self.refresh_interval = config["scheduler"]["check_reindex_days"] + self.integrate_ryd = config["downloads"]["integrate_ryd"] # scan self.all_youtube_ids = False self.all_channel_ids = False @@ -92,6 +93,27 @@ class Reindex: all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]] return all_youtube_ids + def get_unrated_vids(self): + """get all videos without rating if ryd integration is enabled""" + headers = {"Content-type": "application/json"} + data = { + "query": { + "bool": { + "must_not": [{"exists": {"field": "stats.average_rating"}}] + } + } + } + query_str = json.dumps(data) + url = self.es_url + "/ta_video/_search" + response = requests.get( + url, data=query_str, headers=headers, auth=self.es_auth + ) + if not response.ok: + print(response.text) + response_dict = json.loads(response.text) + missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]] + self.all_youtube_ids = self.all_youtube_ids + missing_rating + def get_outdated_channels(self, size): """get daily channels to refresh""" headers = {"Content-type": "application/json"} @@ -156,6 +178,8 @@ class Reindex: self.all_youtube_ids = self.get_outdated_vids(video_daily) self.all_channel_ids = self.get_outdated_channels(channel_daily) self.all_playlist_ids = self.get_outdated_playlists(playlist_daily) + if self.integrate_ryd: + self.get_unrated_vids() def rescrape_all_channels(self): """sync new data from channel to all matching videos""" diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 9eb7731..6e728ea 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -98,8 +98,8 @@

Integrations

-

Integrate with returnyoutubedislike.com: {{ config.downloads.integrate_ryd }}

- Get dislikes and average ratings back.
+

Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}

+ Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.
{{ app_form.downloads_integrate_ryd }}