From 8932a6bc0296466b9c032c62f911180951eed310 Mon Sep 17 00:00:00 2001
From: simon <simobilleter@gmail.com>
Date: Sun, 14 Nov 2021 15:46:24 +0700
Subject: [PATCH] validate existence of playlist entry in index

---
 tubearchivist/home/src/index.py | 8 ++++++--
 tubearchivist/home/tasks.py     | 7 ++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/tubearchivist/home/src/index.py b/tubearchivist/home/src/index.py
index 705a24f..c2310a5 100644
--- a/tubearchivist/home/src/index.py
+++ b/tubearchivist/home/src/index.py
@@ -440,9 +440,10 @@ class YoutubePlaylist:
     ES_URL = CONFIG["application"]["es_url"]
     ES_AUTH = CONFIG["application"]["es_auth"]
 
-    def __init__(self, playlist_id):
+    def __init__(self, playlist_id, all_youtube_ids=False):
         self.playlist_id = playlist_id
         self.stamp = int(datetime.now().strftime("%s"))
+        self.all_youtube_ids = all_youtube_ids
         self.playlist_dict = False
 
     def get_playlist_dict(self, scrape=False):
@@ -514,13 +515,16 @@ class YoutubePlaylist:
         all_members = []
         for idx, entry in enumerate(playlist["entries"]):
             uploader = entry["uploader"]
+            youtube_id = entry["id"]
+            downloaded = youtube_id in self.all_youtube_ids
             if not uploader:
                 continue
             to_append = {
-                "youtube_id": entry["id"],
+                "youtube_id": youtube_id,
                 "title": entry["title"],
                 "uploader": uploader,
                 "idx": idx,
+                "downloaded": downloaded,
             }
             all_members.append(to_append)
 
diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py
index c73aa00..3be1660 100644
--- a/tubearchivist/home/tasks.py
+++ b/tubearchivist/home/tasks.py
@@ -208,9 +208,14 @@ def index_channel_playlists(channel_id):
     channel_handler = YoutubeChannel(channel_id)
     all_playlists = channel_handler.get_all_playlists()
 
+    all_indexed = PendingList().get_all_indexed()
+    all_youtube_ids = [i["_source"]["youtube_id"] for i in all_indexed]
+
     for playlist_id, playlist_title in all_playlists:
         print("add playlist: " + playlist_title)
-        playlist_handler = YoutubePlaylist(playlist_id)
+        playlist_handler = YoutubePlaylist(
+            playlist_id, all_youtube_ids=all_youtube_ids
+        )
         playlist_handler.get_playlist_dict()
         playlist_handler.upload_to_es()
         playlist_handler.add_vids_to_playlist()