From 8932a6bc0296466b9c032c62f911180951eed310 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 14 Nov 2021 15:46:24 +0700 Subject: [PATCH] validate existence of playlist entry in index --- tubearchivist/home/src/index.py | 8 ++++++-- tubearchivist/home/tasks.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tubearchivist/home/src/index.py b/tubearchivist/home/src/index.py index 705a24f..c2310a5 100644 --- a/tubearchivist/home/src/index.py +++ b/tubearchivist/home/src/index.py @@ -440,9 +440,10 @@ class YoutubePlaylist: ES_URL = CONFIG["application"]["es_url"] ES_AUTH = CONFIG["application"]["es_auth"] - def __init__(self, playlist_id): + def __init__(self, playlist_id, all_youtube_ids=False): self.playlist_id = playlist_id self.stamp = int(datetime.now().strftime("%s")) + self.all_youtube_ids = all_youtube_ids self.playlist_dict = False def get_playlist_dict(self, scrape=False): @@ -514,13 +515,16 @@ class YoutubePlaylist: all_members = [] for idx, entry in enumerate(playlist["entries"]): uploader = entry["uploader"] + youtube_id = entry["id"] + downloaded = youtube_id in self.all_youtube_ids if not uploader: continue to_append = { - "youtube_id": entry["id"], + "youtube_id": youtube_id, "title": entry["title"], "uploader": uploader, "idx": idx, + "downloaded": downloaded, } all_members.append(to_append) diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index c73aa00..3be1660 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -208,9 +208,14 @@ def index_channel_playlists(channel_id): channel_handler = YoutubeChannel(channel_id) all_playlists = channel_handler.get_all_playlists() + all_indexed = PendingList().get_all_indexed() + all_youtube_ids = [i["_source"]["youtube_id"] for i in all_indexed] + for playlist_id, playlist_title in all_playlists: print("add playlist: " + playlist_title) - playlist_handler = YoutubePlaylist(playlist_id) + playlist_handler = YoutubePlaylist( + playlist_id, all_youtube_ids=all_youtube_ids + ) playlist_handler.get_playlist_dict() playlist_handler.upload_to_es() playlist_handler.add_vids_to_playlist()