From 4067b6c182afd9525a5391626a3ddd1854563031 Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 15 Apr 2023 22:55:30 +0700 Subject: [PATCH] add channel aggs --- tubearchivist/home/src/frontend/searching.py | 58 ++++--------------- tubearchivist/home/src/index/video_streams.py | 6 +- .../home/templates/home/channel_id.html | 10 ++-- tubearchivist/home/views.py | 12 +++- 4 files changed, 26 insertions(+), 60 deletions(-) diff --git a/tubearchivist/home/src/frontend/searching.py b/tubearchivist/home/src/frontend/searching.py index a082f05..3e7894b 100644 --- a/tubearchivist/home/src/frontend/searching.py +++ b/tubearchivist/home/src/frontend/searching.py @@ -11,6 +11,7 @@ from datetime import datetime from home.src.download.thumbnails import ThumbManager from home.src.es.connect import ElasticWrap +from home.src.index.video_streams import DurationConverter from home.src.ta.config import AppConfig @@ -19,6 +20,7 @@ class SearchHandler: def __init__(self, path, config, data=False): self.max_hits = None + self.aggs = None self.path = path self.config = config self.data = data @@ -34,62 +36,22 @@ class SearchHandler: # simulate list for single result to reuse rest of class return_value = [response] - # stop if empty if not return_value: return False - all_videos = [] - all_channels = [] for idx, hit in enumerate(return_value): return_value[idx] = self.hit_cleanup(hit) - if hit["_index"] == "ta_video": - video_dict, channel_dict = self.vid_cache_link(hit) - if video_dict not in all_videos: - all_videos.append(video_dict) - if channel_dict not in all_channels: - all_channels.append(channel_dict) - elif hit["_index"] == "ta_channel": - channel_dict = self.channel_cache_link(hit) - if channel_dict not in all_channels: - all_channels.append(channel_dict) + + if response.get("aggregations"): + self.aggs = response["aggregations"] + if "total_duration" in self.aggs: + duration_sec = self.aggs["total_duration"]["value"] + self.aggs["total_duration"].update( + {"value_str": DurationConverter().get_str(duration_sec)} + ) return return_value - @staticmethod - def vid_cache_link(hit): - """download thumbnails into cache""" - vid_thumb = hit["source"]["vid_thumb_url"] - youtube_id = hit["source"]["youtube_id"] - channel_id_hit = hit["source"]["channel"]["channel_id"] - chan_thumb = hit["source"]["channel"]["channel_thumb_url"] - try: - chan_banner = hit["source"]["channel"]["channel_banner_url"] - except KeyError: - chan_banner = False - video_dict = {"youtube_id": youtube_id, "vid_thumb": vid_thumb} - channel_dict = { - "channel_id": channel_id_hit, - "chan_thumb": chan_thumb, - "chan_banner": chan_banner, - } - return video_dict, channel_dict - - @staticmethod - def channel_cache_link(hit): - """build channel thumb links""" - channel_id_hit = hit["source"]["channel_id"] - chan_thumb = hit["source"]["channel_thumb_url"] - try: - chan_banner = hit["source"]["channel_banner_url"] - except KeyError: - chan_banner = False - channel_dict = { - "channel_id": channel_id_hit, - "chan_thumb": chan_thumb, - "chan_banner": chan_banner, - } - return channel_dict - @staticmethod def hit_cleanup(hit): """clean up and parse data from a single hit""" diff --git a/tubearchivist/home/src/index/video_streams.py b/tubearchivist/home/src/index/video_streams.py index e477760..5379df9 100644 --- a/tubearchivist/home/src/index/video_streams.py +++ b/tubearchivist/home/src/index/video_streams.py @@ -41,9 +41,9 @@ class DurationConverter: # failed to extract return "NA" - hours = duration_sec // 3600 - minutes = (duration_sec - (hours * 3600)) // 60 - secs = duration_sec - (hours * 3600) - (minutes * 60) + hours = int(duration_sec // 3600) + minutes = int((duration_sec - (hours * 3600)) // 60) + secs = int(duration_sec - (hours * 3600) - (minutes * 60)) duration_str = str() if hours: diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html index fc85d0f..8948772 100644 --- a/tubearchivist/home/templates/home/channel_id.html +++ b/tubearchivist/home/templates/home/channel_id.html @@ -45,12 +45,10 @@
-
- {% if max_hits %} -

Total Videos: {{ max_hits }}

- - {% endif %} -
+ {% if aggs %} +

{{ aggs.total_items.value }} videos | {{ aggs.total_duration.value_str }} playback | Total size {{ aggs.total_size.value|filesizeformat }}

+ + {% endif %}
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py index 527e18b..9f891c4 100644 --- a/tubearchivist/home/views.py +++ b/tubearchivist/home/views.py @@ -148,8 +148,8 @@ class ArchivistViewConfig(View): class ArchivistResultsView(ArchivistViewConfig): """View class to inherit from when searching data in es""" - view_origin = False - es_search = False + view_origin = "" + es_search = "" def __init__(self): super().__init__(self.view_origin) @@ -259,6 +259,7 @@ class ArchivistResultsView(ArchivistViewConfig): self.pagination_handler.validate(search.max_hits) self.context["max_hits"] = search.max_hits self.context["pagination"] = self.pagination_handler.pagination + self.context["aggs"] = search.aggs class MinView(View): @@ -613,6 +614,11 @@ class ChannelIdView(ChannelIdBaseView): ] } } + self.data["aggs"] = { + "total_items": {"value_count": {"field": "youtube_id"}}, + "total_size": {"sum": {"field": "media_size"}}, + "total_duration": {"sum": {"field": "player.duration"}}, + } self.data["sort"].append({"title.keyword": {"order": "asc"}}) if self.context["hide_watched"]: @@ -982,7 +988,7 @@ class SearchView(ArchivistResultsView): """ view_origin = "home" - es_search = False + es_search = "" def get(self, request): """handle get request"""