tubearchivist
/
tubearchivist
mirror of https://github.com/tubearchivist/tubearchivist.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

playlist relation take 2

This commit is contained in:
simon 2021-11-13 17:34:58 +07:00
parent 7930da5242
commit ac14cd5ebb
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
4 changed files with 34 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
tubearchivist/home/src/index.py
View File

@ -535,46 +535,27 @@ class YoutubePlaylist:
print(response.text)
def add_vids_to_playlist(self):
"""sync the playlistdict to video dict"""
print("sync playlist meta data for " + self.playlist_id)
"""sync the playlist id to videos"""
playlist_dict = self.playlist_dict
all_entries = playlist_dict["playlist_entries"]
vid_ids = [i["youtube_id"] for i in all_entries]
to_add = {
key: val
for key, val in playlist_dict.items()
if key != "playlist_entries"
}
bulk_list = []
for youtube_id in vid_ids:
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
source = {"doc": {"playlist": to_add}}
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
# add last newline
bulk_list.append("\n")
query_str = "\n".join(bulk_list)
headers = {"Content-type": "application/x-ndjson"}
url = self.ES_URL + "/_bulk"
response = requests.post(
url, data=query_str, headers=headers, auth=self.ES_AUTH
script = (
'if (!ctx._source.containsKey("playlist")) '
+ "{ctx._source.playlist = [params.playlist]} "
+ "else if (!ctx._source.playlist.contains(params.playlist)) "
+ "{ctx._source.playlist.add(params.playlist)} "
+ "else {ctx.op = 'none'}"
)
if not response.ok:
print(response.text)
def playlist_position(self):
"""sync playlist_position to video dict"""
all_entries = self.playlist_dict["playlist_entries"]
bulk_list = []
for idx, entry in enumerate(all_entries):
for entry in playlist_dict["playlist_entries"]:
youtube_id = entry["youtube_id"]
playlist_position = self.get_position_dict(all_entries, idx)
action = {"update": {"_id": youtube_id, "_index": "ta_video"}}
source = {"doc": {"playlist_position": playlist_position}}
source = {
"script": {
"source": script,
"lang": "painless",
"params": {"playlist": self.playlist_id},
}
}
bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(source))
@ -589,48 +570,6 @@ class YoutubePlaylist:
if not response.ok:
print(response.text)
@staticmethod
def get_position_dict(all_entries, idx):
"""get previous and next videos in playlist"""
playlist_position = {"playlist_index": idx}
if idx == 0:
playlist_position.update(
{
"playlist_prev_id": False,
"playlist_prev_title": False,
"playlist_prev_channel_name": False,
}
)
else:
prev_vid = all_entries[idx - 1]
playlist_position.update(
{
"playlist_prev_id": prev_vid["youtube_id"],
"playlist_prev_title": prev_vid["title"],
"playlist_prev_channel_name": prev_vid["uploader"],
}
)
if idx == len(all_entries) - 1:
playlist_position.update(
{
"playlist_next_id": False,
"playlist_next_title": False,
"playlist_next_channel_name": False,
}
)
else:
next_vid = all_entries[idx + 1]
playlist_position.update(
{
"playlist_next_id": next_vid["youtube_id"],
"playlist_next_title": next_vid["title"],
"playlist_next_channel_name": next_vid["uploader"],
}
)
return playlist_position
class WatchState:
"""handle watched checkbox for videos and channels"""

44
tubearchivist/home/src/index_management.py
View File

@ -107,42 +107,14 @@ INDEX_CONFIG = [
"youtube_id": {"type": "keyword"},
"published": {"type": "date"},
"playlist": {
"properties": {
"playlist_id": {"type": "keyword"},
"playlist_description": {"type": "text"},
"playlist_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower",
}
},
},
"playlist_channel": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower",
}
},
},
"playlist_channel_id": {"type": "keyword"},
"playlist_thumbnail": {"type": "keyword"},
}
},
"playlist_position": {
"properties": {
"playlist_next_id": {"type": "keyword"},
"playlist_next_title": {"type": "text"},
"playlist_next_channel_name": {"type": "text"},
"playlist_prev_id": {"type": "keyword"},
"playlist_prev_title": {"type": "text"},
"playlist_prev_channel_name": {"type": "text"},
}
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"normalizer": "to_lower",
}
},
},
},
"expected_set": {

1
tubearchivist/home/tasks.py
View File

@ -214,7 +214,6 @@ def index_channel_playlists(channel_id):
playlist_handler.get_playlist_dict()
playlist_handler.upload_to_es()
playlist_handler.add_vids_to_playlist()
playlist_handler.playlist_position()
if all_playlists:
handler = ThumbManager()

19
tubearchivist/home/views.py
View File

@ -33,6 +33,7 @@ from home.tasks import (
download_pending,
download_single,
extrac_dl,
index_channel_playlists,
kill_dl,
re_sync_thumbs,
rescan_filesystem,
@ -634,6 +635,7 @@ class PlaylistIdView(View):
def build_data(pagination_handler, playlist_id_detail, view_config):
"""build data query for es"""
sort_by = view_config["sort_by"]
sort_order = view_config["sort_order"]
# overwrite sort_by to match key
if sort_by == "views":
@ -649,17 +651,11 @@ class PlaylistIdView(View):
"query": {
"bool": {
"must": [
{
"term": {
"playlist.playlist_id": {
"value": playlist_id_detail
}
}
}
{"match": {"playlist.keyword": playlist_id_detail}}
]
}
},
"sort": [{"playlist_position.playlist_index": {"order": "asc"}}],
"sort": [{sort_by: {"order": sort_order}}],
}
if view_config["hide_watched"]:
to_append = {"term": {"player.watched": {"value": False}}}
@ -891,6 +887,7 @@ class PostData:
"channel-search": self.channel_search,
"delete-video": self.delete_video,
"delete-channel": self.delete_channel,
"find-playlists": self.find_playlists,
}
return exec_map[self.to_exec]
@ -1082,3 +1079,9 @@ class PostData:
channel_id = self.exec_val
YoutubeChannel(channel_id).delete_channel()
return {"success": True}
def find_playlists(self):
"""add all playlists of a channel"""
channel_id = self.exec_val
index_channel_playlists.delay(channel_id)
return {"success": True}