diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py index 6b9ee33..643ab7c 100644 --- a/tubearchivist/home/src/reindex.py +++ b/tubearchivist/home/src/reindex.py @@ -42,6 +42,7 @@ class Reindex: self.es_url = config["application"]["es_url"] self.es_auth = config["application"]["es_auth"] self.refresh_interval = config["scheduler"]["check_reindex_days"] + self.integrate_ryd = config["downloads"]["integrate_ryd"] # scan self.all_youtube_ids = False self.all_channel_ids = False @@ -92,6 +93,27 @@ class Reindex: all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]] return all_youtube_ids + def get_unrated_vids(self): + """get all videos without rating if ryd integration is enabled""" + headers = {"Content-type": "application/json"} + data = { + "query": { + "bool": { + "must_not": [{"exists": {"field": "stats.average_rating"}}] + } + } + } + query_str = json.dumps(data) + url = self.es_url + "/ta_video/_search" + response = requests.get( + url, data=query_str, headers=headers, auth=self.es_auth + ) + if not response.ok: + print(response.text) + response_dict = json.loads(response.text) + missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]] + self.all_youtube_ids = self.all_youtube_ids + missing_rating + def get_outdated_channels(self, size): """get daily channels to refresh""" headers = {"Content-type": "application/json"} @@ -156,6 +178,8 @@ class Reindex: self.all_youtube_ids = self.get_outdated_vids(video_daily) self.all_channel_ids = self.get_outdated_channels(channel_daily) self.all_playlist_ids = self.get_outdated_playlists(playlist_daily) + if self.integrate_ryd: + self.get_unrated_vids() def rescrape_all_channels(self): """sync new data from channel to all matching videos""" diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html index 9eb7731..6e728ea 100644 --- a/tubearchivist/home/templates/home/settings.html +++ b/tubearchivist/home/templates/home/settings.html @@ -98,8 +98,8 @@
Integrate with returnyoutubedislike.com: {{ config.downloads.integrate_ryd }}
- Get dislikes and average ratings back.Integrate with returnyoutubedislike.com to get dislikes and average ratings back: {{ config.downloads.integrate_ryd }}
+ Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.