From c85be4584608f9a2ede2d95ad9ad5281f84ca047 Mon Sep 17 00:00:00 2001
From: simon <simobilleter@gmail.com>
Date: Wed, 5 Jan 2022 14:43:24 +0700
Subject: [PATCH] extend ryd to refresh when missing average_rating

---
 tubearchivist/home/src/reindex.py             | 24 +++++++++++++++++++
 .../home/templates/home/settings.html         |  4 ++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/tubearchivist/home/src/reindex.py b/tubearchivist/home/src/reindex.py
index 6b9ee33..643ab7c 100644
--- a/tubearchivist/home/src/reindex.py
+++ b/tubearchivist/home/src/reindex.py
@@ -42,6 +42,7 @@ class Reindex:
         self.es_url = config["application"]["es_url"]
         self.es_auth = config["application"]["es_auth"]
         self.refresh_interval = config["scheduler"]["check_reindex_days"]
+        self.integrate_ryd = config["downloads"]["integrate_ryd"]
         # scan
         self.all_youtube_ids = False
         self.all_channel_ids = False
@@ -92,6 +93,27 @@ class Reindex:
         all_youtube_ids = [i["_id"] for i in response_dict["hits"]["hits"]]
         return all_youtube_ids
 
+    def get_unrated_vids(self):
+        """get all videos without rating if ryd integration is enabled"""
+        headers = {"Content-type": "application/json"}
+        data = {
+            "query": {
+                "bool": {
+                    "must_not": [{"exists": {"field": "stats.average_rating"}}]
+                }
+            }
+        }
+        query_str = json.dumps(data)
+        url = self.es_url + "/ta_video/_search"
+        response = requests.get(
+            url, data=query_str, headers=headers, auth=self.es_auth
+        )
+        if not response.ok:
+            print(response.text)
+        response_dict = json.loads(response.text)
+        missing_rating = [i["_id"] for i in response_dict["hits"]["hits"]]
+        self.all_youtube_ids = self.all_youtube_ids + missing_rating
+
     def get_outdated_channels(self, size):
         """get daily channels to refresh"""
         headers = {"Content-type": "application/json"}
@@ -156,6 +178,8 @@ class Reindex:
         self.all_youtube_ids = self.get_outdated_vids(video_daily)
         self.all_channel_ids = self.get_outdated_channels(channel_daily)
         self.all_playlist_ids = self.get_outdated_playlists(playlist_daily)
+        if self.integrate_ryd:
+            self.get_unrated_vids()
 
     def rescrape_all_channels(self):
         """sync new data from channel to all matching videos"""
diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html
index 9eb7731..6e728ea 100644
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@@ -98,8 +98,8 @@
         <div class="settings-group">
             <h2 id="integrations">Integrations</h2>
             <div class="settings-item">
-                <p>Integrate with <a href="https://returnyoutubedislike.com/">returnyoutubedislike.com</a>: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
-                <i>Get dislikes and average ratings back.</i><br>
+                <p>Integrate with <a href="https://returnyoutubedislike.com/">returnyoutubedislike.com</a> to get dislikes and average ratings back: <span class="settings-current">{{ config.downloads.integrate_ryd }}</span></p>
+                <i>Before activating that, make sure you have a scraping sleep interval of at least 3 secs set to avoid ratelimiting issues.</i><br>
                 {{ app_form.downloads_integrate_ryd }}
             </div>
         </div>