From 1cdb9e1ad54be786d66b50f10776f441d16a004f Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 22 Sep 2023 23:54:31 +0700 Subject: [PATCH] refactor find_results use ElasticWrap directly --- tubearchivist/home/src/frontend/searching.py | 94 ------------------- .../home/templates/home/channel.html | 24 ++--- .../home/templates/home/channel_id.html | 22 ++--- .../templates/home/channel_id_playlist.html | 16 ++-- .../home/templates/home/downloads.html | 34 +++---- tubearchivist/home/templates/home/home.html | 24 ++--- .../home/templates/home/playlist.html | 16 ++-- .../home/templates/home/playlist_id.html | 22 ++--- tubearchivist/home/views.py | 20 ++-- 9 files changed, 91 insertions(+), 181 deletions(-) diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/home/src/frontend/searching.py index 068027f..5bcc01d 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/home/src/frontend/searching.py @@ -6,103 +6,9 @@ Functionality: - calculate pagination values """ -import urllib.parse -from datetime import datetime from api.src.search_processor import SearchProcess -from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap -from home.src.ta.helper import get_duration_str - - -class SearchHandler: - """search elastic search""" - - def __init__(self, path, data=False): - self.max_hits = None - self.aggs = None - self.path = path - self.data = data - - def get_data(self): - """get the data""" - response, _ = ElasticWrap(self.path).get(self.data) - - if "hits" in response.keys(): - self.max_hits = response["hits"]["total"]["value"] - return_value = response["hits"]["hits"] - else: - # simulate list for single result to reuse rest of class - return_value = [response] - - if not return_value: - return False - - for idx, hit in enumerate(return_value): - return_value[idx] = self.hit_cleanup(hit) - - if response.get("aggregations"): - self.aggs = response["aggregations"] - if "total_duration" in self.aggs: - duration_sec = int(self.aggs["total_duration"]["value"]) - self.aggs["total_duration"].update( - {"value_str": get_duration_str(duration_sec)} - ) - - return return_value - - @staticmethod - def hit_cleanup(hit): - """clean up and parse data from a single hit""" - hit["source"] = hit.pop("_source") - hit_keys = hit["source"].keys() - if "media_url" in hit_keys: - parsed_url = urllib.parse.quote(hit["source"]["media_url"]) - hit["source"]["media_url"] = parsed_url - - if "published" in hit_keys: - published = hit["source"]["published"] - date_pub = datetime.strptime(published, "%Y-%m-%d") - date_str = datetime.strftime(date_pub, "%d %b, %Y") - hit["source"]["published"] = date_str - - if "vid_last_refresh" in hit_keys: - vid_last_refresh = hit["source"]["vid_last_refresh"] - date_refresh = datetime.fromtimestamp(vid_last_refresh) - date_str = datetime.strftime(date_refresh, "%d %b, %Y") - hit["source"]["vid_last_refresh"] = date_str - - if "playlist_last_refresh" in hit_keys: - playlist_last_refresh = hit["source"]["playlist_last_refresh"] - date_refresh = datetime.fromtimestamp(playlist_last_refresh) - date_str = datetime.strftime(date_refresh, "%d %b, %Y") - hit["source"]["playlist_last_refresh"] = date_str - - if "vid_thumb_url" in hit_keys: - youtube_id = hit["source"]["youtube_id"] - thumb_path = ThumbManager(youtube_id).vid_thumb_path() - hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}" - - if "channel_last_refresh" in hit_keys: - refreshed = hit["source"]["channel_last_refresh"] - date_refresh = datetime.fromtimestamp(refreshed) - date_str = datetime.strftime(date_refresh, "%d %b, %Y") - hit["source"]["channel_last_refresh"] = date_str - - if "channel" in hit_keys: - channel_keys = hit["source"]["channel"].keys() - if "channel_last_refresh" in channel_keys: - refreshed = hit["source"]["channel"]["channel_last_refresh"] - date_refresh = datetime.fromtimestamp(refreshed) - date_str = datetime.strftime(date_refresh, "%d %b, %Y") - hit["source"]["channel"]["channel_last_refresh"] = date_str - - if "subtitle_fragment_id" in hit_keys: - youtube_id = hit["source"]["youtube_id"] - thumb_path = ThumbManager(youtube_id).vid_thumb_path() - hit["source"]["vid_thumb_url"] = f"/cache/{thumb_path}" - - return hit class SearchForm: diff --git a/tubearchivist/home/templates/home/channel.html b/tubearchivist/home/templates/home/channel.html index 7a7debf..0091345 100644 --- a/tubearchivist/home/templates/home/channel.html +++ b/tubearchivist/home/templates/home/channel.html @@ -42,33 +42,33 @@ {% for channel in results %}
- - {{ channel.source.channel_id }}-banner + + {{ channel.channel_id }}-banner
- - channel-thumb + + channel-thumb
-

{{ channel.source.channel_name }}

- {% if channel.source.channel_subs >= 1000000 %} -

Subscribers: {{ channel.source.channel_subs|intword }}

+

{{ channel.channel_name }}

+ {% if channel.channel_subs >= 1000000 %} +

Subscribers: {{ channel.channel_subs|intword }}

{% else %} -

Subscribers: {{ channel.source.channel_subs|intcomma }}

+

Subscribers: {{ channel.channel_subs|intcomma }}

{% endif %}
-

Last refreshed: {{ channel.source.channel_last_refresh }}

- {% if channel.source.channel_subscribed %} - +

Last refreshed: {{ channel.channel_last_refresh }}

+ {% if channel.channel_subscribed %} + {% else %} - + {% endif %}
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index 421392c..838615b 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -106,14 +106,14 @@ {% if results %} {% for video in results %}
- +
- video-thumb - {% if video.source.player.progress %} -
+ video-thumb + {% if video.player.progress %} +
{% else %} -
+
{% endif %}
@@ -122,16 +122,16 @@
-
- {% if video.source.player.watched %} - seen-icon +
+ {% if video.player.watched %} + seen-icon {% else %} - unseen-icon + unseen-icon {% endif %} - {{ video.source.published }} | {{ video.source.player.duration_str }} + {{ video.published }} | {{ video.player.duration_str }}
diff --git a/tubearchivist/home/templates/home/channel_id_playlist.html b/tubearchivist/home/templates/home/channel_id_playlist.html index 86d635d..2a199d0 100644 --- a/tubearchivist/home/templates/home/channel_id_playlist.html +++ b/tubearchivist/home/templates/home/channel_id_playlist.html @@ -45,18 +45,18 @@ {% for playlist in results %}
- - {{ playlist.source.playlist_id }}-thumbnail + + {{ playlist.playlist_id }}-thumbnail
-

{{ playlist.source.playlist_channel }}

 -

{{ playlist.source.playlist_name }}

 -

Last refreshed: {{ playlist.source.playlist_last_refresh }}

- {% if playlist.source.playlist_subscribed %} - +

{{ playlist.playlist_channel }}

 +

{{ playlist.playlist_name }}

 +

Last refreshed: {{ playlist.playlist_last_refresh }}

+ {% if playlist.playlist_subscribed %} + {% else %} - + {% endif %}
diff --git a/tubearchivist/home/templates/home/downloads.html b/tubearchivist/home/templates/home/downloads.html index 2084358..16e1f15 100644 --- a/tubearchivist/home/templates/home/downloads.html +++ b/tubearchivist/home/templates/home/downloads.html @@ -70,18 +70,18 @@
{% if results %} {% for video in results %} -
+
- video_thumb + video_thumb
{% if show_ignored_only %} ignored {% else %} queued {% endif %} - {{ video.source.vid_type }} - {% if video.source.auto_start %} + {{ video.vid_type }} + {% if video.auto_start %} auto {% endif %}
@@ -89,27 +89,27 @@
- {% if video.source.channel_indexed %} - {{ video.source.channel_name }} + {% if video.channel_indexed %} + {{ video.channel_name }} {% else %} - {{ video.source.channel_name }} + {{ video.channel_name }} {% endif %} -

{{ video.source.title }}

 +

{{ video.title }}
-

Published: {{ video.source.published }} | Duration: {{ video.source.duration }} | {{ video.source.youtube_id }}

- {% if video.source.message %} -

{{ video.source.message }}

+

Published: {{ video.published }} | Duration: {{ video.duration }} | {{ video.youtube_id }}

+ {% if video.message %} +

{{ video.message }}

{% endif %}
{% if show_ignored_only %} - - + + {% else %} - - + + {% endif %} - {% if video.source.message %} - + {% if video.message %} + {% endif %}
diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html index ade51aa..59a3478 100644 --- a/tubearchivist/home/templates/home/home.html +++ b/tubearchivist/home/templates/home/home.html @@ -95,14 +95,14 @@ {% if results %} {% for video in results %}
- +
- video-thumb - {% if video.source.player.progress %} -
+ video-thumb + {% if video.player.progress %} +
{% else %} -
+
{% endif %}
@@ -111,17 +111,17 @@
-
- {% if video.source.player.watched %} - seen-icon +
+ {% if video.player.watched %} + seen-icon {% else %} - unseen-icon + unseen-icon {% endif %} - {{ video.source.published }} | {{ video.source.player.duration_str }} + {{ video.published }} | {{ video.player.duration_str }}
diff --git a/tubearchivist/home/templates/home/playlist.html b/tubearchivist/home/templates/home/playlist.html index c82af74..c6e0203 100644 --- a/tubearchivist/home/templates/home/playlist.html +++ b/tubearchivist/home/templates/home/playlist.html @@ -40,18 +40,18 @@ {% for playlist in results %}
- - {{ playlist.source.playlist_id }}-thumbnail + + {{ playlist.playlist_id }}-thumbnail
-

{{ playlist.source.playlist_channel }}

 -

{{ playlist.source.playlist_name }}

 -

Last refreshed: {{ playlist.source.playlist_last_refresh }}

- {% if playlist.source.playlist_subscribed %} - +

{{ playlist.playlist_channel }}

 +

{{ playlist.playlist_name }}

 +

Last refreshed: {{ playlist.playlist_last_refresh }}

+ {% if playlist.playlist_subscribed %} + {% else %} - + {% endif %}
diff --git a/tubearchivist/home/templates/home/playlist_id.html b/tubearchivist/home/templates/home/playlist_id.html index 525e99a..3faf449 100644 --- a/tubearchivist/home/templates/home/playlist_id.html +++ b/tubearchivist/home/templates/home/playlist_id.html @@ -110,14 +110,14 @@ {% if results %} {% for video in results %}
- +
- video-thumb - {% if video.source.player.progress %} -
+ video-thumb + {% if video.player.progress %} +
{% else %} -
+
{% endif %}
@@ -126,16 +126,16 @@
-
- {% if video.source.player.watched %} - seen-icon +
+ {% if video.player.watched %} + seen-icon {% else %} - unseen-icon + unseen-icon {% endif %} - {{ video.source.published }} | {{ video.source.player.duration_str }} + {{ video.published }} | {{ video.player.duration_str }}
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 50aebc7..0b6c6a6 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -32,7 +32,6 @@ from home.src.frontend.forms import ( SubscribeToPlaylistForm, UserSettingsForm, ) -from home.src.frontend.searching import SearchHandler from home.src.index.channel import channel_overwrites from home.src.index.generic import Pagination from home.src.index.playlist import YoutubePlaylist @@ -200,12 +199,17 @@ class ArchivistResultsView(ArchivistViewConfig): def find_results(self): """add results and pagination to context""" - search = SearchHandler(self.es_search, data=self.data) - self.context["results"] = search.get_data() - self.pagination_handler.validate(search.max_hits) - self.context["max_hits"] = search.max_hits - self.context["pagination"] = self.pagination_handler.pagination - self.context["aggs"] = search.aggs + response, _ = ElasticWrap(self.es_search).get(self.data) + max_hits = response["hits"]["total"]["value"] + self.pagination_handler.validate(max_hits) + self.context.update( + { + "results": [i["_source"] for i in response["hits"]["hits"]], + "max_hits": max_hits, + "pagination": self.pagination_handler.pagination, + "aggs": response.get("aggregations"), + } + ) class MinView(View): @@ -499,7 +503,7 @@ class ChannelIdView(ChannelIdBaseView): self.channel_pages(channel_id) if self.context["results"]: - channel_info = self.context["results"][0]["source"]["channel"] + channel_info = self.context["results"][0]["channel"] channel_name = channel_info["channel_name"] else: # fall back channel lookup if no videos found