From 832259ce48e095227e827eb122635f7eb60a4b25 Mon Sep 17 00:00:00 2001
From: spechter <hendrik@sadlack.de>
Date: Wed, 15 Nov 2023 05:06:51 +0100
Subject: [PATCH 01/21] Expanded sorting functionality (#589)

* - Added duration and filesize as options in sorting menu on Home and ChannelId views
- Added keys 'duration' and 'filesize' as valid parameters to sort by
- Mapped 'duration' and 'filesize' to their corresponding es keys

* Fixed spelling

* Changed formatting to comply to maximum line length.

* Locally running "deploy.sh validate" before committing

---------

Co-authored-by: spechter <spechter@spechter.net>
---
 tubearchivist/home/src/ta/users.py                | 9 ++++++++-
 tubearchivist/home/templates/home/channel_id.html | 2 ++
 tubearchivist/home/templates/home/home.html       | 2 ++
 tubearchivist/home/views.py                       | 2 ++
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tubearchivist/home/src/ta/users.py b/tubearchivist/home/src/ta/users.py
index bb6a387..7181b3f 100644
--- a/tubearchivist/home/src/ta/users.py
+++ b/tubearchivist/home/src/ta/users.py
@@ -50,7 +50,14 @@ class UserConfig:
     VALID_STYLESHEETS = get_stylesheets()
     VALID_VIEW_STYLE = ["grid", "list"]
     VALID_SORT_ORDER = ["asc", "desc"]
-    VALID_SORT_BY = ["published", "downloaded", "views", "likes"]
+    VALID_SORT_BY = [
+        "published",
+        "downloaded",
+        "views",
+        "likes",
+        "duration",
+        "filesize",
+    ]
     VALID_GRID_ITEMS = range(3, 8)
 
     def __init__(self, user_id: str):
diff --git a/tubearchivist/home/templates/home/channel_id.html b/tubearchivist/home/templates/home/channel_id.html
index c51fbd2..6c8b653 100644
--- a/tubearchivist/home/templates/home/channel_id.html
+++ b/tubearchivist/home/templates/home/channel_id.html
@@ -82,6 +82,8 @@
                     <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                     <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                     <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                 </select>
                 <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                     <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
diff --git a/tubearchivist/home/templates/home/home.html b/tubearchivist/home/templates/home/home.html
index 0724381..f55d241 100644
--- a/tubearchivist/home/templates/home/home.html
+++ b/tubearchivist/home/templates/home/home.html
@@ -65,6 +65,8 @@
                     <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                     <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                     <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                 </select>
                 <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                     <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
diff --git a/tubearchivist/home/views.py b/tubearchivist/home/views.py
index b81895d..6ef8de5 100644
--- a/tubearchivist/home/views.py
+++ b/tubearchivist/home/views.py
@@ -111,6 +111,8 @@ class ArchivistResultsView(ArchivistViewConfig):
             "likes": "stats.like_count",
             "downloaded": "date_downloaded",
             "published": "published",
+            "duration": "player.duration",
+            "filesize": "media_size",
         }
         sort_by = sort_by_map[self.context["sort_by"]]
 

From 2f62898a10f127a0238a12b0e80bd599f225735a Mon Sep 17 00:00:00 2001
From: Derek Slenk <derek.slenk@gmail.com>
Date: Tue, 14 Nov 2023 23:44:43 -0500
Subject: [PATCH 02/21] Add new css item for web footer (#598)

---
 tubearchivist/static/css/style.css | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tubearchivist/static/css/style.css b/tubearchivist/static/css/style.css
index fbd84f6..5b15bf1 100644
--- a/tubearchivist/static/css/style.css
+++ b/tubearchivist/static/css/style.css
@@ -180,6 +180,10 @@ button:hover {
     text-decoration: underline;
 }
 
+.footer .boxed-content {
+    text-align: center;
+}
+
 /* toggle on-off */
 .toggle {
     display: flex;

From 2b49af962059195b1edc4de2fbc533c319de682e Mon Sep 17 00:00:00 2001
From: crocs <34723037+pairofcrocs@users.noreply.github.com>
Date: Tue, 14 Nov 2023 22:45:08 -0600
Subject: [PATCH 03/21] Update settings.html (#599)

This was really bugging me lol
---
 tubearchivist/home/templates/home/settings.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html
index d28d2a7..52a2693 100644
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@@ -5,7 +5,7 @@
     <h1>Your Archive</h1>
 </div>
 <div class="settings-item">
-    <h2>Main overview</h2>
+    <h2>Main Overview</h2>
     <div id="primaryBox" class="info-box info-box-4">
         <p id="loading">Loading...</p>
     </div>

From 6721d01fa62a43b481801c4e973b995fa333c53d Mon Sep 17 00:00:00 2001
From: lamusmaser <1940060+lamusmaser@users.noreply.github.com>
Date: Thu, 16 Nov 2023 19:12:02 -0700
Subject: [PATCH 04/21] Fix `textarea` type from `shell` to `Shell`. (#594)

---
 .github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml b/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
index 38e4fa2..2102c6a 100644
--- a/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
+++ b/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
@@ -38,6 +38,6 @@ body:
     attributes:
       label: Relevant log output
       description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      render: Shell
     validations:
       required: true

From 37df9b65c74609511fc588a3a004ccb737bce36d Mon Sep 17 00:00:00 2001
From: lamusmaser <1940060+lamusmaser@users.noreply.github.com>
Date: Thu, 16 Nov 2023 19:16:09 -0700
Subject: [PATCH 05/21] Add `allowed_null_keys` and its dictionary for manual
 imports. (#595)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add `allowed_null_keys` and its dictionary for manual imports.

* Fix linting for `allowed_null_keys` list.

* Add missing trailing comma for linting.

* Add missing newline that wasn't in earlier linting responses.

* Clear empty text in newlines.

* Remove newline that the linter requested because the linter now doesn't want it. ¯\_(ツ)_/¯

* Change default application from manual import to the video processing.

* Fix missing space.
---
 tubearchivist/home/src/index/video.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py
index b414350..7a997cf 100644
--- a/tubearchivist/home/src/index/video.py
+++ b/tubearchivist/home/src/index/video.py
@@ -188,11 +188,11 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
-            "description": self.youtube_meta["description"],
-            "category": self.youtube_meta["categories"],
+            "description": self.youtube_meta.get("description", ""),
+            "category": self.youtube_meta.get("categories", []),
             "vid_thumb_url": self.youtube_meta["thumbnail"],
             "vid_thumb_base64": base64_blur,
-            "tags": self.youtube_meta["tags"],
+            "tags": self.youtube_meta.get("tags", []),
             "published": published,
             "vid_last_refresh": last_refresh,
             "date_downloaded": last_refresh,

From 669bc6a6207ef0e702a5927d09e57988db4b8166 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Fri, 17 Nov 2023 09:22:11 +0700
Subject: [PATCH 06/21] fallback for view_count, refac, #581

---
 tubearchivist/home/src/index/video.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/tubearchivist/home/src/index/video.py b/tubearchivist/home/src/index/video.py
index 7a997cf..d06ce5e 100644
--- a/tubearchivist/home/src/index/video.py
+++ b/tubearchivist/home/src/index/video.py
@@ -210,20 +210,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def _add_stats(self):
         """add stats dicst to json_data"""
-        # likes
-        like_count = self.youtube_meta.get("like_count", 0)
-        dislike_count = self.youtube_meta.get("dislike_count", 0)
-        average_rating = self.youtube_meta.get("average_rating", 0)
-        self.json_data.update(
-            {
-                "stats": {
-                    "view_count": self.youtube_meta["view_count"],
-                    "like_count": like_count,
-                    "dislike_count": dislike_count,
-                    "average_rating": average_rating,
-                }
-            }
-        )
+        stats = {
+            "view_count": self.youtube_meta.get("view_count", 0),
+            "like_count": self.youtube_meta.get("like_count", 0),
+            "dislike_count": self.youtube_meta.get("dislike_count", 0),
+            "average_rating": self.youtube_meta.get("average_rating", 0),
+        }
+        self.json_data.update({"stats": stats})
 
     def build_dl_cache_path(self):
         """find video path in dl cache"""

From eced8200c198ef436d72b3e93b5f861866a5ae0f Mon Sep 17 00:00:00 2001
From: crocs <34723037+pairofcrocs@users.noreply.github.com>
Date: Thu, 16 Nov 2023 20:23:19 -0600
Subject: [PATCH 07/21] Update settings_scheduling.html (#601)

I found more!
---
 tubearchivist/home/templates/home/settings_scheduling.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tubearchivist/home/templates/home/settings_scheduling.html b/tubearchivist/home/templates/home/settings_scheduling.html
index b3b57a3..1b011c1 100644
--- a/tubearchivist/home/templates/home/settings_scheduling.html
+++ b/tubearchivist/home/templates/home/settings_scheduling.html
@@ -51,7 +51,7 @@
         </div>
     </div>
     <div class="settings-group">
-        <h2>Start download</h2>
+        <h2>Start Download</h2>
         <div class="settings-item">
             <p>Current Download schedule: <span class="settings-current">
                 {% if config.scheduler.download_pending %}
@@ -112,7 +112,7 @@
         </div>
     </div>
     <div class="settings-group">
-        <h2>Thumbnail check</h2>
+        <h2>Thumbnail Check</h2>
         <div class="settings-item">
             <p>Current thumbnail check schedule: <span class="settings-current">
                 {% if config.scheduler.thumbnail_check %}

From f41ecd24c518fe45690b5a4117db450005c0701e Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Fri, 17 Nov 2023 09:26:31 +0700
Subject: [PATCH 08/21] fix missing config for comments extraction, #596

---
 tubearchivist/home/src/index/comments.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubearchivist/home/src/index/comments.py b/tubearchivist/home/src/index/comments.py
index 5f25602..50e0f17 100644
--- a/tubearchivist/home/src/index/comments.py
+++ b/tubearchivist/home/src/index/comments.py
@@ -77,7 +77,7 @@ class Comments:
     def get_yt_comments(self):
         """get comments from youtube"""
         yt_obs = self.build_yt_obs()
-        info_json = YtWrap(yt_obs).extract(self.youtube_id)
+        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
         if not info_json:
             return False, False
 

From 9a9d35cac47151b500afab0ee544478ed0061041 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Fri, 17 Nov 2023 09:44:10 +0700
Subject: [PATCH 09/21] explicitly define player mapping, #592

---
 tubearchivist/home/src/es/index_mapping.json | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json
index d6dda4b..7d486c7 100644
--- a/tubearchivist/home/src/es/index_mapping.json
+++ b/tubearchivist/home/src/es/index_mapping.json
@@ -252,6 +252,24 @@
                         }
                     }
                 },
+                "player": {
+                    "properties": {
+                        "duration": {
+                            "type": "long"
+                        },
+                        "duration_str": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "watched": {
+                            "type": "boolean"
+                        },
+                        "watched_date": {
+                            "type": "date",
+                            "format": "epoch_second"
+                        }
+                    }
+                },
                 "subtitles": {
                     "properties": {
                         "ext": {

From 463019ce5a12b6f4b2fa8258ccd584e1e8a150a6 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sat, 18 Nov 2023 17:30:31 +0700
Subject: [PATCH 10/21] fix outdated redinex now_lte datatype

---
 tubearchivist/home/src/index/reindex.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py
index e87b30a..64b8813 100644
--- a/tubearchivist/home/src/index/reindex.py
+++ b/tubearchivist/home/src/index/reindex.py
@@ -123,7 +123,7 @@ class ReindexPopulate(ReindexBase):
         """get outdated from index_name"""
         index_name = reindex_config["index_name"]
         refresh_key = reindex_config["refresh_key"]
-        now_lte = self.now - self.interval * 24 * 60 * 60
+        now_lte = str(self.now - self.interval * 24 * 60 * 60)
         must_list = [
             {"match": {reindex_config["active_key"]: True}},
             {"range": {refresh_key: {"lte": now_lte}}},

From 4a145ee7cb9f523326c50614e33ba0577a2c92ac Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sat, 18 Nov 2023 17:44:16 +0700
Subject: [PATCH 11/21] paginate to get total active docs count

---
 tubearchivist/home/src/index/reindex.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tubearchivist/home/src/index/reindex.py b/tubearchivist/home/src/index/reindex.py
index 64b8813..53b62d1 100644
--- a/tubearchivist/home/src/index/reindex.py
+++ b/tubearchivist/home/src/index/reindex.py
@@ -105,11 +105,13 @@ class ReindexPopulate(ReindexBase):
         """get total hits from index"""
         index_name = reindex_config["index_name"]
         active_key = reindex_config["active_key"]
-        path = f"{index_name}/_search?filter_path=hits.total"
-        data = {"query": {"match": {active_key: True}}}
-        response, _ = ElasticWrap(path).post(data=data)
-        total_hits = response["hits"]["total"]["value"]
-        return total_hits
+        data = {
+            "query": {"term": {active_key: {"value": True}}},
+            "_source": False,
+        }
+        total = IndexPaginate(index_name, data, keep_source=True).get_results()
+
+        return len(total)
 
     def _get_daily_should(self, total_hits):
         """calc how many should reindex daily"""

From 91bb0ed9c03d44024c3fc2b0e0dd6b95525caf43 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 13:01:27 +0700
Subject: [PATCH 12/21] [API] add video aggregation

---
 tubearchivist/api/src/aggs.py | 52 +++++++++++++++++++++++++++++++++++
 tubearchivist/api/urls.py     |  6 ++--
 tubearchivist/api/views.py    | 10 +++----
 3 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index 8921d86..f26e295 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -84,6 +84,58 @@ class Primary(AggBase):
         return response
 
 
+class Video(AggBase):
+    """get video stats"""
+
+    name = "video_stats"
+    path = "ta_video/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "video_type": {
+                "terms": {"field": "vid_type"},
+                "aggs": {"media_size": {"sum": {"field": "media_size"}}},
+            },
+            "video_active": {
+                "terms": {"field": "active"},
+                "aggs": {"media_size": {"sum": {"field": "media_size"}}},
+            },
+            "video_media_size": {"sum": {"field": "media_size"}},
+            "video_count": {"value_count": {"field": "youtube_id"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+
+        response = {
+            "doc_count": aggregations["video_count"]["value"],
+            "media_size": int(aggregations["video_media_size"]["value"]),
+        }
+        for bucket in aggregations["video_type"]["buckets"]:
+            response.update(
+                {
+                    f"type_{bucket['key']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                    }
+                }
+            )
+
+        for bucket in aggregations["video_active"]["buckets"]:
+            response.update(
+                {
+                    f"active_{bucket['key_as_string']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                    }
+                }
+            )
+
+        return response
+
+
 class WatchProgress(AggBase):
     """get watch progress"""
 
diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py
index 8b075f2..3f06802 100644
--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@@ -152,9 +152,9 @@ urlpatterns = [
         name="api-notification",
     ),
     path(
-        "stats/primary/",
-        views.StatPrimaryView.as_view(),
-        name="api-stats-primary",
+        "stats/video/",
+        views.StatVideoView.as_view(),
+        name="api-stats-video",
     ),
     path(
         "stats/watch/",
diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py
index 0e64d10..e2cfe69 100644
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@@ -1,6 +1,6 @@
 """all API views"""
 
-from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress
+from api.src.aggs import BiggestChannel, DownloadHist, Video, WatchProgress
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
@@ -1141,16 +1141,16 @@ class NotificationView(ApiBaseView):
         return Response(RedisArchivist().list_items(query))
 
 
-class StatPrimaryView(ApiBaseView):
-    """resolves to /api/stats/primary/
-    GET: return document count
+class StatVideoView(ApiBaseView):
+    """resolves to /api/stats/video/
+    GET: return video stats
     """
 
     def get(self, request):
         """get stats"""
         # pylint: disable=unused-argument
 
-        return Response(Primary().process())
+        return Response(Video().process())
 
 
 class StatWatchProgress(ApiBaseView):

From b1267cba8323277a6e79d86a7fc7d3450e71f2a2 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 13:06:47 +0700
Subject: [PATCH 13/21] standard json style

---
 tubearchivist/home/src/es/index_mapping.json | 60 ++++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/tubearchivist/home/src/es/index_mapping.json b/tubearchivist/home/src/es/index_mapping.json
index 7d486c7..52ee884 100644
--- a/tubearchivist/home/src/es/index_mapping.json
+++ b/tubearchivist/home/src/es/index_mapping.json
@@ -74,7 +74,7 @@
                             "type": "boolean"
                         },
                         "integrate_sponsorblock": {
-                            "type" : "boolean"
+                            "type": "boolean"
                         }
                     }
                 }
@@ -168,7 +168,7 @@
                                     "type": "boolean"
                                 },
                                 "integrate_sponsorblock": {
-                                    "type" : "boolean"
+                                    "type": "boolean"
                                 }
                             }
                         }
@@ -236,19 +236,19 @@
                 "comment_count": {
                     "type": "long"
                 },
-                "stats" : {
-                    "properties" : {
-                        "average_rating" : {
-                            "type" : "float"
+                "stats": {
+                    "properties": {
+                        "average_rating": {
+                            "type": "float"
                         },
-                        "dislike_count" : {
-                            "type" : "long"
+                        "dislike_count": {
+                            "type": "long"
                         },
-                        "like_count" : {
-                            "type" : "long"
+                        "like_count": {
+                            "type": "long"
                         },
-                        "view_count" : {
-                            "type" : "long"
+                        "view_count": {
+                            "type": "long"
                         }
                     }
                 },
@@ -332,28 +332,28 @@
                         "is_enabled": {
                             "type": "boolean"
                         },
-                        "segments" : {
-                            "properties" : {
-                                "UUID" : {
+                        "segments": {
+                            "properties": {
+                                "UUID": {
                                     "type": "keyword"
                                 },
-                                "actionType" : {
+                                "actionType": {
                                     "type": "keyword"
                                 },
-                                "category" : {
+                                "category": {
                                     "type": "keyword"
                                 },
-                                "locked" : {
-                                    "type" : "short"
+                                "locked": {
+                                    "type": "short"
                                 },
-                                "segment" : {
-                                    "type" : "float"
+                                "segment": {
+                                    "type": "float"
                                 },
-                                "videoDuration" : {
-                                    "type" : "float"
+                                "videoDuration": {
+                                    "type": "float"
                                 },
-                                "votes" : {
-                                    "type" : "long"
+                                "votes": {
+                                    "type": "long"
                                 }
                             }
                         }
@@ -534,7 +534,7 @@
                     "format": "epoch_second"
                 },
                 "subtitle_index": {
-                    "type" : "long"
+                    "type": "long"
                 },
                 "subtitle_lang": {
                     "type": "keyword"
@@ -543,7 +543,7 @@
                     "type": "keyword"
                 },
                 "subtitle_line": {
-                    "type" : "text",
+                    "type": "text",
                     "analyzer": "english"
                 }
             },
@@ -578,14 +578,14 @@
                             "type": "keyword"
                         },
                         "comment_text": {
-                            "type" : "text"
+                            "type": "text"
                         },
                         "comment_timestamp": {
                             "type": "date",
                             "format": "epoch_second"
                         },
                         "comment_time_text": {
-                            "type" : "text"
+                            "type": "text"
                         },
                         "comment_likecount": {
                             "type": "long"
@@ -631,4 +631,4 @@
             }
         }
     ]
-}
+}
\ No newline at end of file

From e74c26fe36c54e01b6a3e99b45349443fb0f02ef Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 13:48:24 +0700
Subject: [PATCH 14/21] [API] add channel aggs

---
 tubearchivist/api/src/aggs.py | 31 +++++++++++++++++++++++++++++++
 tubearchivist/api/urls.py     |  5 +++++
 tubearchivist/api/views.py    | 20 +++++++++++++++++++-
 3 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index f26e295..1374842 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -136,6 +136,37 @@ class Video(AggBase):
         return response
 
 
+class Channel(AggBase):
+    """get channel stats"""
+
+    name = "channel_stats"
+    path = "ta_channel/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "channel_count": {"value_count": {"field": "channel_id"}},
+            "channel_active": {"terms": {"field": "channel_active"}},
+            "channel_subscribed": {"terms": {"field": "channel_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+
+        response = {
+            "doc_count": aggregations["channel_count"].get("value"),
+        }
+        for bucket in aggregations["channel_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["channel_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
 class WatchProgress(AggBase):
     """get watch progress"""
 
diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py
index 3f06802..bc61460 100644
--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@@ -156,6 +156,11 @@ urlpatterns = [
         views.StatVideoView.as_view(),
         name="api-stats-video",
     ),
+    path(
+        "stats/channel/",
+        views.StatChannelView.as_view(),
+        name="api-stats-channel",
+    ),
     path(
         "stats/watch/",
         views.StatWatchProgress.as_view(),
diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py
index e2cfe69..de09089 100644
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@@ -1,6 +1,12 @@
 """all API views"""
 
-from api.src.aggs import BiggestChannel, DownloadHist, Video, WatchProgress
+from api.src.aggs import (
+    BiggestChannel,
+    Channel,
+    DownloadHist,
+    Video,
+    WatchProgress,
+)
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
@@ -1153,6 +1159,18 @@ class StatVideoView(ApiBaseView):
         return Response(Video().process())
 
 
+class StatChannelView(ApiBaseView):
+    """resolves to /api/stats/channel/
+    GET: return channel stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Channel().process())
+
+
 class StatWatchProgress(ApiBaseView):
     """resolves to /api/stats/watchprogress/
     GET: return watch/unwatch progress stats

From a466c02304a9b051efbce8f52b86fe9b1b53de42 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 14:00:27 +0700
Subject: [PATCH 15/21] [API] add playlist stats

---
 tubearchivist/api/src/aggs.py | 28 ++++++++++++++++++++++++++++
 tubearchivist/api/urls.py     |  5 +++++
 tubearchivist/api/views.py    | 13 +++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index 1374842..989c04f 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -167,6 +167,34 @@ class Channel(AggBase):
         return response
 
 
+class Playlist(AggBase):
+    """get playlist stats"""
+
+    name = "playlist_stats"
+    path = "ta_playlist/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "playlist_count": {"value_count": {"field": "playlist_id"}},
+            "playlist_active": {"terms": {"field": "playlist_active"}},
+            "playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {"doc_count": aggregations["playlist_count"].get("value")}
+        for bucket in aggregations["playlist_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["playlist_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
 class WatchProgress(AggBase):
     """get watch progress"""
 
diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py
index bc61460..8422181 100644
--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@@ -161,6 +161,11 @@ urlpatterns = [
         views.StatChannelView.as_view(),
         name="api-stats-channel",
     ),
+    path(
+        "stats/playlist/",
+        views.StatPlaylistView.as_view(),
+        name="api-stats-playlist",
+    ),
     path(
         "stats/watch/",
         views.StatWatchProgress.as_view(),
diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py
index de09089..fbde800 100644
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@@ -4,6 +4,7 @@ from api.src.aggs import (
     BiggestChannel,
     Channel,
     DownloadHist,
+    Playlist,
     Video,
     WatchProgress,
 )
@@ -1171,6 +1172,18 @@ class StatChannelView(ApiBaseView):
         return Response(Channel().process())
 
 
+class StatPlaylistView(ApiBaseView):
+    """resolves to /api/stats/playlist/
+    GET: return playlist stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Playlist().process())
+
+
 class StatWatchProgress(ApiBaseView):
     """resolves to /api/stats/watchprogress/
     GET: return watch/unwatch progress stats

From 02ac590caa87c55935657d9f71dd447bf22cadab Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 14:42:16 +0700
Subject: [PATCH 16/21] [API] add download stats

---
 tubearchivist/api/src/aggs.py | 30 ++++++++++++++++++++++++++++++
 tubearchivist/api/urls.py     |  5 +++++
 tubearchivist/api/views.py    | 13 +++++++++++++
 3 files changed, 48 insertions(+)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index 989c04f..0895bcb 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -195,6 +195,36 @@ class Playlist(AggBase):
         return response
 
 
+class Download(AggBase):
+    """get downloads queue stats"""
+
+    name = "download_queue_stats"
+    path = "ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "status": {"terms": {"field": "status"}},
+            "video_type": {
+                "filter": {"term": {"status": "pending"}},
+                "aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
+            },
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {}
+        for bucket in aggregations["status"]["buckets"]:
+            response.update({bucket["key"]: bucket.get("doc_count")})
+
+        for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
+            key = f"pending_{bucket['key']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
 class WatchProgress(AggBase):
     """get watch progress"""
 
diff --git a/tubearchivist/api/urls.py b/tubearchivist/api/urls.py
index 8422181..b03547c 100644
--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@@ -166,6 +166,11 @@ urlpatterns = [
         views.StatPlaylistView.as_view(),
         name="api-stats-playlist",
     ),
+    path(
+        "stats/download/",
+        views.StatDownloadView.as_view(),
+        name="api-stats-download",
+    ),
     path(
         "stats/watch/",
         views.StatWatchProgress.as_view(),
diff --git a/tubearchivist/api/views.py b/tubearchivist/api/views.py
index fbde800..e026dba 100644
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@@ -3,6 +3,7 @@
 from api.src.aggs import (
     BiggestChannel,
     Channel,
+    Download,
     DownloadHist,
     Playlist,
     Video,
@@ -1184,6 +1185,18 @@ class StatPlaylistView(ApiBaseView):
         return Response(Playlist().process())
 
 
+class StatDownloadView(ApiBaseView):
+    """resolves to /api/stats/download/
+    GET: return download stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Download().process())
+
+
 class StatWatchProgress(ApiBaseView):
     """resolves to /api/stats/watchprogress/
     GET: return watch/unwatch progress stats

From 44c4cf93e2e74c8e99daf1579802efab706baac9 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 20:27:18 +0700
Subject: [PATCH 17/21] refactor dashboard tile building

---
 .../home/templates/home/settings.html         |   5 +-
 tubearchivist/static/stats.js                 | 286 ++++++++++--------
 2 files changed, 171 insertions(+), 120 deletions(-)

diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html
index 52a2693..93f3e74 100644
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@@ -5,10 +5,13 @@
     <h1>Your Archive</h1>
 </div>
 <div class="settings-item">
-    <h2>Main Overview</h2>
+    <h2>Overview</h2>
     <div id="primaryBox" class="info-box info-box-4">
         <p id="loading">Loading...</p>
     </div>
+    <div id="secondaryBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
 </div>
 <div class="settings-item">
     <h2>Watch Progress</h2>
diff --git a/tubearchivist/static/stats.js b/tubearchivist/static/stats.js
index d575ea4..d85955a 100644
--- a/tubearchivist/static/stats.js
+++ b/tubearchivist/static/stats.js
@@ -5,158 +5,140 @@
 /* globals apiRequest */
 
 function primaryStats() {
-  let apiEndpoint = '/api/stats/primary/';
-  let responseData = apiRequest(apiEndpoint, 'GET');
-  let primaryBox = document.getElementById('primaryBox');
+  let apiVideoEndpoint = '/api/stats/video/';
+  let responseData = apiRequest(apiVideoEndpoint, 'GET');
 
+  let primaryBox = document.getElementById('primaryBox');
   clearLoading(primaryBox);
 
-  let videoTile = buildVideoTile(responseData);
-  primaryBox.appendChild(videoTile);
+  let totalTile = buildTotalVideoTile(responseData);
+  primaryBox.appendChild(totalTile);
 
-  let channelTile = buildChannelTile(responseData);
-  primaryBox.appendChild(channelTile);
+  let videosTypeTile = buildVideosTypeTile(responseData);
+  primaryBox.appendChild(videosTypeTile);
 
-  let playlistTile = buildPlaylistTile(responseData);
-  primaryBox.appendChild(playlistTile);
+  let shortsTypeTile = buildShortsTypeTile(responseData);
+  primaryBox.appendChild(shortsTypeTile);
 
-  let downloadTile = buildDownloadTile(responseData);
-  primaryBox.appendChild(downloadTile);
+  let streamsTypeTile = buildStreamsTypeTile(responseData);
+  primaryBox.appendChild(streamsTypeTile);
 }
 
-function clearLoading(dashBox) {
-  dashBox.querySelector('#loading').remove();
+function secondaryStats() {
+  let apiChannelEndpoint = '/api/stats/channel/';
+  let channelResponseData = apiRequest(apiChannelEndpoint, 'GET');
+  let secondaryBox = document.getElementById('secondaryBox');
+  clearLoading(secondaryBox);
+  let channelTile = buildChannelTile(channelResponseData);
+  secondaryBox.appendChild(channelTile);
+
+  let apiPlaylistEndpoint = '/api/stats/playlist/';
+  let playlistResponseData = apiRequest(apiPlaylistEndpoint, 'GET');
+  let playlistTile = buildPlaylistTile(playlistResponseData);
+  secondaryBox.appendChild(playlistTile);
+
+  let apiDownloadEndpoint = '/api/stats/download/';
+  let downloadResponseData = apiRequest(apiDownloadEndpoint, 'GET');
+  let downloadTile = buildDownloadTile(downloadResponseData);
+  secondaryBox.appendChild(downloadTile);
 }
 
-function buildTile(titleText) {
-  let tile = document.createElement('div');
-  tile.classList.add('info-box-item');
-
-  let title = document.createElement('h3');
-
-  title.innerText = titleText;
-  tile.appendChild(title);
-
+function buildTotalVideoTile(responseData) {
+  const totalCount = responseData.doc_count || 0;
+  const totalSize = humanFileSize(responseData.media_size || 0);
+  const content = {
+    Items: `${totalCount}`,
+    'Media Size': `${totalSize}`,
+  };
+  const tile = buildTile('All: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
   return tile;
 }
 
-function buildTileContenTable(content, rowsWanted) {
-  let contentEntries = Object.entries(content);
-
-  const nbsp = '\u00A0'; // No-Break Space https://www.compart.com/en/unicode/U+00A0
-
-  // Do not add spacing rows when on mobile device
-  const isMobile = window.matchMedia('(max-width: 600px)');
-  if (!isMobile.matches) {
-    if (contentEntries.length < rowsWanted) {
-      const rowsToAdd = rowsWanted - contentEntries.length;
-
-      for (let i = 0; i < rowsToAdd; i++) {
-        contentEntries.push([nbsp, nbsp]);
-      }
-    }
-  }
-
-  const table = document.createElement('table');
-  table.classList.add('agg-channel-table');
-  const tableBody = document.createElement('tbody');
-
-  for (const [key, value] of contentEntries) {
-    const row = document.createElement('tr');
-
-    const leftCell = document.createElement('td');
-    leftCell.classList.add('agg-channel-name');
-
-    // Do not add ":" when its a spacing entry
-    const keyText = key === nbsp ? key : `${key}: `;
-    const leftText = document.createTextNode(keyText);
-    leftCell.appendChild(leftText);
-
-    const rightCell = document.createElement('td');
-    rightCell.classList.add('agg-channel-right-align');
-
-    const rightText = document.createTextNode(value);
-    rightCell.appendChild(rightText);
-
-    row.appendChild(leftCell);
-    row.appendChild(rightCell);
-
-    tableBody.appendChild(row);
-  }
-
-  table.appendChild(tableBody);
-
-  return table;
+function buildVideosTypeTile(responseData) {
+  const videosCount = responseData.type_videos.doc_count || 0;
+  const videosSize = humanFileSize(responseData.type_videos.media_size || 0);
+  const content = {
+    Items: `${videosCount}`,
+    'Media Size': `${videosSize}`,
+  };
+  const tile = buildTile('Regular Videos: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
 }
 
-function buildVideoTile(responseData) {
-  let tile = buildTile(`Video types: `);
-
-  const total = responseData.videos.total || 0;
-  const videos = responseData.videos.videos || 0;
-  const shorts = responseData.videos.shorts || 0;
-  const streams = responseData.videos.streams || 0;
-
+function buildShortsTypeTile(responseData) {
+  const shortsCount = responseData.type_shorts.doc_count || 0;
+  const shortsSize = humanFileSize(responseData.type_shorts.media_size || 0);
   const content = {
-    Videos: `${videos}/${total}`,
-    Shorts: `${shorts}/${total}`,
-    Streams: `${streams}/${total}`,
+    Items: `${shortsCount}`,
+    'Media Size': `${shortsSize}`,
   };
-
-  const table = buildTileContenTable(content, 3);
-
+  const tile = buildTile('Shorts: ');
+  const table = buildTileContenTable(content, 2);
   tile.appendChild(table);
+  return tile;
+}
 
+function buildStreamsTypeTile(responseData) {
+  const streamsCount = responseData.type_streams.doc_count || 0;
+  const streamsSize = humanFileSize(responseData.type_streams.media_size || 0);
+  const content = {
+    Items: `${streamsCount}`,
+    'Media Size': `${streamsSize}`,
+  };
+  const tile = buildTile('Streams: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
   return tile;
 }
 
 function buildChannelTile(responseData) {
-  let tile = buildTile(`Channels: `);
-
-  const total = responseData.channels.total || 0;
-  const subscribed = responseData.channels.sub_true || 0;
-
+  let tile = buildTile('Channels: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
   const content = {
-    Subscribed: `${subscribed}/${total}`,
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
   };
-
   const table = buildTileContenTable(content, 3);
-
   tile.appendChild(table);
 
   return tile;
 }
 
 function buildPlaylistTile(responseData) {
-  let tile = buildTile(`Playlists:`);
-
-  const total = responseData.playlists.total || 0;
-  const subscribed = responseData.playlists.sub_true || 0;
-
+  let tile = buildTile('Playlists: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
   const content = {
-    Subscribed: `${subscribed}/${total}`,
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
   };
-
-  const table = buildTileContenTable(content, 3);
-
+  const table = buildTileContenTable(content, 2);
   tile.appendChild(table);
 
   return tile;
 }
 
 function buildDownloadTile(responseData) {
-  let tile = buildTile('Downloads');
-
-  const pending = responseData.downloads.pending || 0;
-  const ignored = responseData.downloads.ignore || 0;
-
+  const pendingTotal = responseData.pending || 0;
+  let tile = buildTile(`Downloads Pending: ${pendingTotal}`);
+  const pendingVideos = responseData.pending_videos || 0;
+  const pendingShorts = responseData.pending_shorts || 0;
+  const pendingStreams = responseData.pending_streams || 0;
   const content = {
-    Pending: pending,
-    Ignored: ignored,
+    Videos: pendingVideos,
+    Shorts: pendingShorts,
+    Streams: pendingStreams,
   };
-
   const table = buildTileContenTable(content, 3);
-
   tile.appendChild(table);
 
   return tile;
@@ -180,11 +162,6 @@ function watchStats() {
   watchBox.appendChild(thirdCard);
 }
 
-function capitalizeFirstLetter(string) {
-  // source: https://stackoverflow.com/a/1026087
-  return string.charAt(0).toUpperCase() + string.slice(1);
-}
-
 function buildWatchTile(title, watchDetail) {
   const items = watchDetail?.items ?? 0;
   const duration = watchDetail?.duration ?? 0;
@@ -248,11 +225,6 @@ function buildDailyStat(dailyStat) {
   return tile;
 }
 
-function humanFileSize(size) {
-  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
-  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
-}
-
 function buildChannelRow(id, name, value) {
   let tableRow = document.createElement('tr');
 
@@ -309,6 +281,81 @@ function addBiggestChannelByMediaSize() {
   }
 }
 
+function clearLoading(dashBox) {
+  dashBox.querySelector('#loading').remove();
+}
+
+function capitalizeFirstLetter(string) {
+  // source: https://stackoverflow.com/a/1026087
+  return string.charAt(0).toUpperCase() + string.slice(1);
+}
+
+function humanFileSize(size) {
+  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
+  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
+function buildTile(titleText) {
+  let tile = document.createElement('div');
+  tile.classList.add('info-box-item');
+
+  let title = document.createElement('h3');
+
+  title.innerText = titleText;
+  tile.appendChild(title);
+
+  return tile;
+}
+
+function buildTileContenTable(content, rowsWanted) {
+  let contentEntries = Object.entries(content);
+
+  const nbsp = '\u00A0'; // No-Break Space https://www.compart.com/en/unicode/U+00A0
+
+  // Do not add spacing rows when on mobile device
+  const isMobile = window.matchMedia('(max-width: 600px)');
+  if (!isMobile.matches) {
+    if (contentEntries.length < rowsWanted) {
+      const rowsToAdd = rowsWanted - contentEntries.length;
+
+      for (let i = 0; i < rowsToAdd; i++) {
+        contentEntries.push([nbsp, nbsp]);
+      }
+    }
+  }
+
+  const table = document.createElement('table');
+  table.classList.add('agg-channel-table');
+  const tableBody = document.createElement('tbody');
+
+  for (const [key, value] of contentEntries) {
+    const row = document.createElement('tr');
+
+    const leftCell = document.createElement('td');
+    leftCell.classList.add('agg-channel-name');
+
+    // Do not add ":" when its a spacing entry
+    const keyText = key === nbsp ? key : `${key}: `;
+    const leftText = document.createTextNode(keyText);
+    leftCell.appendChild(leftText);
+
+    const rightCell = document.createElement('td');
+    rightCell.classList.add('agg-channel-right-align');
+
+    const rightText = document.createTextNode(value);
+    rightCell.appendChild(rightText);
+
+    row.appendChild(leftCell);
+    row.appendChild(rightCell);
+
+    tableBody.appendChild(row);
+  }
+
+  table.appendChild(tableBody);
+
+  return table;
+}
+
 function biggestChannel() {
   addBiggestChannelByDocCount();
   addBiggestChannelByDuration();
@@ -317,6 +364,7 @@ function biggestChannel() {
 
 async function buildStats() {
   primaryStats();
+  secondaryStats();
   watchStats();
   downloadHist();
   biggestChannel();

From d5676e51732d21494fd8395532cfffaf50116192 Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 20:30:50 +0700
Subject: [PATCH 18/21] [API] remove primary endpoint, in favor of dedicated
 stats

---
 tubearchivist/api/src/aggs.py | 60 -----------------------------------
 1 file changed, 60 deletions(-)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index 0895bcb..e800753 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -24,66 +24,6 @@ class AggBase:
         raise NotImplementedError
 
 
-class Primary(AggBase):
-    """primary aggregation for total documents indexed"""
-
-    name = "primary"
-    path = "ta_video,ta_channel,ta_playlist,ta_subtitle,ta_download/_search"
-    data = {
-        "size": 0,
-        "aggs": {
-            "video_type": {
-                "filter": {"exists": {"field": "active"}},
-                "aggs": {"filtered": {"terms": {"field": "vid_type"}}},
-            },
-            "channel_total": {"value_count": {"field": "channel_active"}},
-            "channel_sub": {"terms": {"field": "channel_subscribed"}},
-            "playlist_total": {"value_count": {"field": "playlist_active"}},
-            "playlist_sub": {"terms": {"field": "playlist_subscribed"}},
-            "download": {"terms": {"field": "status"}},
-        },
-    }
-
-    def process(self):
-        """make the call"""
-        aggregations = self.get()
-
-        videos = {"total": aggregations["video_type"].get("doc_count")}
-        videos.update(
-            {
-                i.get("key"): i.get("doc_count")
-                for i in aggregations["video_type"]["filtered"]["buckets"]
-            }
-        )
-        channels = {"total": aggregations["channel_total"].get("value")}
-        channels.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["channel_sub"]["buckets"]
-            }
-        )
-        playlists = {"total": aggregations["playlist_total"].get("value")}
-        playlists.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["playlist_sub"]["buckets"]
-            }
-        )
-        downloads = {
-            i.get("key"): i.get("doc_count")
-            for i in aggregations["download"]["buckets"]
-        }
-
-        response = {
-            "videos": videos,
-            "channels": channels,
-            "playlists": playlists,
-            "downloads": downloads,
-        }
-
-        return response
-
-
 class Video(AggBase):
     """get video stats"""
 

From a369be0f4ab3e4a6ab082fd1c6fa36adb9d69b8e Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 21:20:42 +0700
Subject: [PATCH 19/21] split active videos tile, add duration

---
 tubearchivist/api/src/aggs.py                 | 20 +++++-
 .../home/templates/home/settings.html         | 13 +++-
 tubearchivist/static/stats.js                 | 68 ++++++++++++++-----
 3 files changed, 79 insertions(+), 22 deletions(-)

diff --git a/tubearchivist/api/src/aggs.py b/tubearchivist/api/src/aggs.py
index e800753..e2c65db 100644
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@@ -34,14 +34,21 @@ class Video(AggBase):
         "aggs": {
             "video_type": {
                 "terms": {"field": "vid_type"},
-                "aggs": {"media_size": {"sum": {"field": "media_size"}}},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
             },
             "video_active": {
                 "terms": {"field": "active"},
-                "aggs": {"media_size": {"sum": {"field": "media_size"}}},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
             },
             "video_media_size": {"sum": {"field": "media_size"}},
             "video_count": {"value_count": {"field": "youtube_id"}},
+            "duration": {"sum": {"field": "player.duration"}},
         },
     }
 
@@ -49,26 +56,35 @@ class Video(AggBase):
         """process aggregation"""
         aggregations = self.get()
 
+        duration = int(aggregations["duration"]["value"])
         response = {
             "doc_count": aggregations["video_count"]["value"],
             "media_size": int(aggregations["video_media_size"]["value"]),
+            "duration": duration,
+            "duration_str": get_duration_str(duration),
         }
         for bucket in aggregations["video_type"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
             response.update(
                 {
                     f"type_{bucket['key']}": {
                         "doc_count": bucket.get("doc_count"),
                         "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
                     }
                 }
             )
 
         for bucket in aggregations["video_active"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
             response.update(
                 {
                     f"active_{bucket['key_as_string']}": {
                         "doc_count": bucket.get("doc_count"),
                         "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
                     }
                 }
             )
diff --git a/tubearchivist/home/templates/home/settings.html b/tubearchivist/home/templates/home/settings.html
index 93f3e74..e72f735 100644
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@@ -6,16 +6,25 @@
 </div>
 <div class="settings-item">
     <h2>Overview</h2>
-    <div id="primaryBox" class="info-box info-box-4">
+    <div id="activeBox" class="info-box info-box-3">
         <p id="loading">Loading...</p>
     </div>
+</div>
+<div class="settings-item">
+    <h2>Video Type</h2>
+    <div id="videoTypeBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Application</h2>
     <div id="secondaryBox" class="info-box info-box-3">
         <p id="loading">Loading...</p>
     </div>
 </div>
 <div class="settings-item">
     <h2>Watch Progress</h2>
-    <div id="watchBox" class="info-box info-box-3">
+    <div id="watchBox" class="info-box info-box-2">
         <p id="loading">Loading...</p>
     </div>
 </div>
diff --git a/tubearchivist/static/stats.js b/tubearchivist/static/stats.js
index d85955a..208c246 100644
--- a/tubearchivist/static/stats.js
+++ b/tubearchivist/static/stats.js
@@ -8,20 +8,25 @@ function primaryStats() {
   let apiVideoEndpoint = '/api/stats/video/';
   let responseData = apiRequest(apiVideoEndpoint, 'GET');
 
-  let primaryBox = document.getElementById('primaryBox');
-  clearLoading(primaryBox);
+  let activeBox = document.getElementById('activeBox');
+  clearLoading(activeBox);
 
   let totalTile = buildTotalVideoTile(responseData);
-  primaryBox.appendChild(totalTile);
+  activeBox.appendChild(totalTile);
+  let activeTile = buildActiveVideoTile(responseData);
+  activeBox.appendChild(activeTile);
+  let inActiveTile = buildInActiveVideoTile(responseData);
+  activeBox.appendChild(inActiveTile);
+
+  let videoTypeBox = document.getElementById('videoTypeBox');
+  clearLoading(videoTypeBox);
 
   let videosTypeTile = buildVideosTypeTile(responseData);
-  primaryBox.appendChild(videosTypeTile);
-
+  videoTypeBox.appendChild(videosTypeTile);
   let shortsTypeTile = buildShortsTypeTile(responseData);
-  primaryBox.appendChild(shortsTypeTile);
-
+  videoTypeBox.appendChild(shortsTypeTile);
   let streamsTypeTile = buildStreamsTypeTile(responseData);
-  primaryBox.appendChild(streamsTypeTile);
+  videoTypeBox.appendChild(streamsTypeTile);
 }
 
 function secondaryStats() {
@@ -49,6 +54,7 @@ function buildTotalVideoTile(responseData) {
   const content = {
     Items: `${totalCount}`,
     'Media Size': `${totalSize}`,
+    Duration: responseData.duration_str,
   };
   const tile = buildTile('All: ');
   const table = buildTileContenTable(content, 2);
@@ -56,12 +62,41 @@ function buildTotalVideoTile(responseData) {
   return tile;
 }
 
+function buildActiveVideoTile(responseData) {
+  const activeCount = responseData.active_true.doc_count || 0;
+  const activeSize = humanFileSize(responseData.active_true.media_size) || 0;
+  const content = {
+    Items: `${activeCount}`,
+    'Media Size': `${activeSize}`,
+    Duration: responseData.active_true.duration_str,
+  };
+  const tile = buildTile('Active: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildInActiveVideoTile(responseData) {
+  const inActiveCount = responseData.active_false.doc_count || 0;
+  const inActiveSize = humanFileSize(responseData.active_false.media_size) || 0;
+  const content = {
+    Items: `${inActiveCount}`,
+    'Media Size': `${inActiveSize}`,
+    Duration: responseData.active_false.duration_str,
+  };
+  const tile = buildTile('Inactive: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
 function buildVideosTypeTile(responseData) {
   const videosCount = responseData.type_videos.doc_count || 0;
   const videosSize = humanFileSize(responseData.type_videos.media_size || 0);
   const content = {
     Items: `${videosCount}`,
     'Media Size': `${videosSize}`,
+    Duration: responseData.type_videos.duration_str,
   };
   const tile = buildTile('Regular Videos: ');
   const table = buildTileContenTable(content, 2);
@@ -75,6 +110,7 @@ function buildShortsTypeTile(responseData) {
   const content = {
     Items: `${shortsCount}`,
     'Media Size': `${shortsSize}`,
+    Duration: responseData.type_shorts.duration_str,
   };
   const tile = buildTile('Shorts: ');
   const table = buildTileContenTable(content, 2);
@@ -88,6 +124,7 @@ function buildStreamsTypeTile(responseData) {
   const content = {
     Items: `${streamsCount}`,
     'Media Size': `${streamsSize}`,
+    Duration: responseData.type_streams.duration_str,
   };
   const tile = buildTile('Streams: ');
   const table = buildTileContenTable(content, 2);
@@ -150,16 +187,11 @@ function watchStats() {
   let watchBox = document.getElementById('watchBox');
   clearLoading(watchBox);
 
-  const { total, watched, unwatched } = responseData;
+  let watchedTile = buildWatchTile('watched', responseData.watched);
+  watchBox.appendChild(watchedTile);
 
-  let firstCard = buildWatchTile('total', total);
-  watchBox.appendChild(firstCard);
-
-  let secondCard = buildWatchTile('watched', watched);
-  watchBox.appendChild(secondCard);
-
-  let thirdCard = buildWatchTile('unwatched', unwatched);
-  watchBox.appendChild(thirdCard);
+  let unwatchedTile = buildWatchTile('unwatched', responseData.unwatched);
+  watchBox.appendChild(unwatchedTile);
 }
 
 function buildWatchTile(title, watchDetail) {
@@ -180,7 +212,7 @@ function buildWatchTile(title, watchDetail) {
   const content = {
     Videos: items,
     Seconds: duration,
-    Playback: duration_str,
+    Duration: duration_str,
   };
 
   const table = buildTileContenTable(content, 3);

From 96e73a3a5369d11d636dbca3884961f36cb939bc Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 21:32:11 +0700
Subject: [PATCH 20/21] handle empty tile response

---
 tubearchivist/static/stats.js | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/tubearchivist/static/stats.js b/tubearchivist/static/stats.js
index 208c246..4200d74 100644
--- a/tubearchivist/static/stats.js
+++ b/tubearchivist/static/stats.js
@@ -63,12 +63,13 @@ function buildTotalVideoTile(responseData) {
 }
 
 function buildActiveVideoTile(responseData) {
-  const activeCount = responseData.active_true.doc_count || 0;
-  const activeSize = humanFileSize(responseData.active_true.media_size) || 0;
+  const activeCount = responseData?.active_true?.doc_count || 0;
+  const activeSize = humanFileSize(responseData?.active_true?.media_size || 0);
+  const duration = responseData?.active_true?.duration_str || 'NA';
   const content = {
     Items: `${activeCount}`,
     'Media Size': `${activeSize}`,
-    Duration: responseData.active_true.duration_str,
+    Duration: duration,
   };
   const tile = buildTile('Active: ');
   const table = buildTileContenTable(content, 2);
@@ -77,12 +78,13 @@ function buildActiveVideoTile(responseData) {
 }
 
 function buildInActiveVideoTile(responseData) {
-  const inActiveCount = responseData.active_false.doc_count || 0;
-  const inActiveSize = humanFileSize(responseData.active_false.media_size) || 0;
+  const inActiveCount = responseData?.active_false?.doc_count || 0;
+  const inActiveSize = humanFileSize(responseData?.active_false?.media_size || 0);
+  const duration = responseData?.active_false?.duration_str || 'NA';
   const content = {
     Items: `${inActiveCount}`,
     'Media Size': `${inActiveSize}`,
-    Duration: responseData.active_false.duration_str,
+    Duration: duration,
   };
   const tile = buildTile('Inactive: ');
   const table = buildTileContenTable(content, 2);
@@ -91,12 +93,13 @@ function buildInActiveVideoTile(responseData) {
 }
 
 function buildVideosTypeTile(responseData) {
-  const videosCount = responseData.type_videos.doc_count || 0;
-  const videosSize = humanFileSize(responseData.type_videos.media_size || 0);
+  const videosCount = responseData?.type_videos?.doc_count || 0;
+  const videosSize = humanFileSize(responseData?.type_videos?.media_size || 0);
+  const duration = responseData?.type_videos?.duration_str || 'NA';
   const content = {
     Items: `${videosCount}`,
     'Media Size': `${videosSize}`,
-    Duration: responseData.type_videos.duration_str,
+    Duration: duration,
   };
   const tile = buildTile('Regular Videos: ');
   const table = buildTileContenTable(content, 2);
@@ -105,12 +108,13 @@ function buildVideosTypeTile(responseData) {
 }
 
 function buildShortsTypeTile(responseData) {
-  const shortsCount = responseData.type_shorts.doc_count || 0;
-  const shortsSize = humanFileSize(responseData.type_shorts.media_size || 0);
+  const shortsCount = responseData?.type_shorts?.doc_count || 0;
+  const shortsSize = humanFileSize(responseData?.type_shorts?.media_size || 0);
+  const duration = responseData?.type_shorts?.duration_str || 'NA';
   const content = {
     Items: `${shortsCount}`,
     'Media Size': `${shortsSize}`,
-    Duration: responseData.type_shorts.duration_str,
+    Duration: duration,
   };
   const tile = buildTile('Shorts: ');
   const table = buildTileContenTable(content, 2);
@@ -119,12 +123,13 @@ function buildShortsTypeTile(responseData) {
 }
 
 function buildStreamsTypeTile(responseData) {
-  const streamsCount = responseData.type_streams.doc_count || 0;
-  const streamsSize = humanFileSize(responseData.type_streams.media_size || 0);
+  const streamsCount = responseData?.type_streams?.doc_count || 0;
+  const streamsSize = humanFileSize(responseData?.type_streams?.media_size || 0);
+  const duration = responseData?.type_streams?.duration_str || 'NA';
   const content = {
     Items: `${streamsCount}`,
     'Media Size': `${streamsSize}`,
-    Duration: responseData.type_streams.duration_str,
+    Duration: duration,
   };
   const tile = buildTile('Streams: ');
   const table = buildTileContenTable(content, 2);

From 2e4289e75c0605ef6d473c99eb9c0bd6fd3aa29b Mon Sep 17 00:00:00 2001
From: Simon <simobilleter@gmail.com>
Date: Sun, 19 Nov 2023 21:37:08 +0700
Subject: [PATCH 21/21] bump requirements

---
 tubearchivist/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tubearchivist/requirements.txt b/tubearchivist/requirements.txt
index a36184c..d5aee90 100644
--- a/tubearchivist/requirements.txt
+++ b/tubearchivist/requirements.txt
@@ -1,8 +1,8 @@
 apprise==1.6.0
-celery==5.3.4
+celery==5.3.5
 Django==4.2.7
 django-auth-ldap==4.6.0
-django-cors-headers==4.3.0
+django-cors-headers==4.3.1
 djangorestframework==3.14.0
 Pillow==10.1.0
 redis==5.0.1
@@ -10,4 +10,4 @@ requests==2.31.0
 ryd-client==0.0.6
 uWSGI==2.0.23
 whitenoise==6.6.0
-yt-dlp==2023.10.13
+yt-dlp==2023.11.16