From ac14cd5ebbd2a9a182e59b1fb1a89d416a74cfa1 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 13 Nov 2021 17:34:58 +0700 Subject: [PATCH] playlist relation take 2 --- tubearchivist/home/src/index.py | 91 ++++------------------ tubearchivist/home/src/index_management.py | 44 ++--------- tubearchivist/home/tasks.py | 1 - tubearchivist/home/views.py | 19 +++-- 4 files changed, 34 insertions(+), 121 deletions(-) diff --git a/tubearchivist/home/src/index.py b/tubearchivist/home/src/index.py index b6595c9..705a24f 100644 --- a/tubearchivist/home/src/index.py +++ b/tubearchivist/home/src/index.py @@ -535,46 +535,27 @@ class YoutubePlaylist: print(response.text) def add_vids_to_playlist(self): - """sync the playlistdict to video dict""" - print("sync playlist meta data for " + self.playlist_id) + """sync the playlist id to videos""" playlist_dict = self.playlist_dict - all_entries = playlist_dict["playlist_entries"] - vid_ids = [i["youtube_id"] for i in all_entries] - - to_add = { - key: val - for key, val in playlist_dict.items() - if key != "playlist_entries" - } - bulk_list = [] - for youtube_id in vid_ids: - action = {"update": {"_id": youtube_id, "_index": "ta_video"}} - source = {"doc": {"playlist": to_add}} - bulk_list.append(json.dumps(action)) - bulk_list.append(json.dumps(source)) - - # add last newline - bulk_list.append("\n") - query_str = "\n".join(bulk_list) - - headers = {"Content-type": "application/x-ndjson"} - url = self.ES_URL + "/_bulk" - response = requests.post( - url, data=query_str, headers=headers, auth=self.ES_AUTH + script = ( + 'if (!ctx._source.containsKey("playlist")) ' + + "{ctx._source.playlist = [params.playlist]} " + + "else if (!ctx._source.playlist.contains(params.playlist)) " + + "{ctx._source.playlist.add(params.playlist)} " + + "else {ctx.op = 'none'}" ) - if not response.ok: - print(response.text) - - def playlist_position(self): - """sync playlist_position to video dict""" - all_entries = self.playlist_dict["playlist_entries"] bulk_list = [] - for idx, entry in enumerate(all_entries): + for entry in playlist_dict["playlist_entries"]: youtube_id = entry["youtube_id"] - playlist_position = self.get_position_dict(all_entries, idx) action = {"update": {"_id": youtube_id, "_index": "ta_video"}} - source = {"doc": {"playlist_position": playlist_position}} + source = { + "script": { + "source": script, + "lang": "painless", + "params": {"playlist": self.playlist_id}, + } + } bulk_list.append(json.dumps(action)) bulk_list.append(json.dumps(source)) @@ -589,48 +570,6 @@ class YoutubePlaylist: if not response.ok: print(response.text) - @staticmethod - def get_position_dict(all_entries, idx): - """get previous and next videos in playlist""" - playlist_position = {"playlist_index": idx} - if idx == 0: - playlist_position.update( - { - "playlist_prev_id": False, - "playlist_prev_title": False, - "playlist_prev_channel_name": False, - } - ) - else: - prev_vid = all_entries[idx - 1] - playlist_position.update( - { - "playlist_prev_id": prev_vid["youtube_id"], - "playlist_prev_title": prev_vid["title"], - "playlist_prev_channel_name": prev_vid["uploader"], - } - ) - - if idx == len(all_entries) - 1: - playlist_position.update( - { - "playlist_next_id": False, - "playlist_next_title": False, - "playlist_next_channel_name": False, - } - ) - else: - next_vid = all_entries[idx + 1] - playlist_position.update( - { - "playlist_next_id": next_vid["youtube_id"], - "playlist_next_title": next_vid["title"], - "playlist_next_channel_name": next_vid["uploader"], - } - ) - - return playlist_position - class WatchState: """handle watched checkbox for videos and channels""" diff --git a/tubearchivist/home/src/index_management.py b/tubearchivist/home/src/index_management.py index acd06f9..d7d7a21 100644 --- a/tubearchivist/home/src/index_management.py +++ b/tubearchivist/home/src/index_management.py @@ -107,42 +107,14 @@ INDEX_CONFIG = [ "youtube_id": {"type": "keyword"}, "published": {"type": "date"}, "playlist": { - "properties": { - "playlist_id": {"type": "keyword"}, - "playlist_description": {"type": "text"}, - "playlist_name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "to_lower", - } - }, - }, - "playlist_channel": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256, - "normalizer": "to_lower", - } - }, - }, - "playlist_channel_id": {"type": "keyword"}, - "playlist_thumbnail": {"type": "keyword"}, - } - }, - "playlist_position": { - "properties": { - "playlist_next_id": {"type": "keyword"}, - "playlist_next_title": {"type": "text"}, - "playlist_next_channel_name": {"type": "text"}, - "playlist_prev_id": {"type": "keyword"}, - "playlist_prev_title": {"type": "text"}, - "playlist_prev_channel_name": {"type": "text"}, - } + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "to_lower", + } + }, }, }, "expected_set": { diff --git a/tubearchivist/home/tasks.py b/tubearchivist/home/tasks.py index e86212c..c73aa00 100644 --- a/tubearchivist/home/tasks.py +++ b/tubearchivist/home/tasks.py @@ -214,7 +214,6 @@ def index_channel_playlists(channel_id): playlist_handler.get_playlist_dict() playlist_handler.upload_to_es() playlist_handler.add_vids_to_playlist() - playlist_handler.playlist_position() if all_playlists: handler = ThumbManager() diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 1b5e16d..e51c158 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -33,6 +33,7 @@ from home.tasks import ( download_pending, download_single, extrac_dl, + index_channel_playlists, kill_dl, re_sync_thumbs, rescan_filesystem, @@ -634,6 +635,7 @@ class PlaylistIdView(View): def build_data(pagination_handler, playlist_id_detail, view_config): """build data query for es""" sort_by = view_config["sort_by"] + sort_order = view_config["sort_order"] # overwrite sort_by to match key if sort_by == "views": @@ -649,17 +651,11 @@ class PlaylistIdView(View): "query": { "bool": { "must": [ - { - "term": { - "playlist.playlist_id": { - "value": playlist_id_detail - } - } - } + {"match": {"playlist.keyword": playlist_id_detail}} ] } }, - "sort": [{"playlist_position.playlist_index": {"order": "asc"}}], + "sort": [{sort_by: {"order": sort_order}}], } if view_config["hide_watched"]: to_append = {"term": {"player.watched": {"value": False}}} @@ -891,6 +887,7 @@ class PostData: "channel-search": self.channel_search, "delete-video": self.delete_video, "delete-channel": self.delete_channel, + "find-playlists": self.find_playlists, } return exec_map[self.to_exec] @@ -1082,3 +1079,9 @@ class PostData: channel_id = self.exec_val YoutubeChannel(channel_id).delete_channel() return {"success": True} + + def find_playlists(self): + """add all playlists of a channel""" + channel_id = self.exec_val + index_channel_playlists.delay(channel_id) + return {"success": True}