Improved dashboard, reindex fix, #build

Changed: - Added additional sort by fields - [API] Chaned primary stats endpoints - [API] Added separate video stats endpoints - Added fallback for some manual import values - Fix comment extration for members video - Fix reindex outdated query
2023-11-19 22:02:23 +07:00 · 2023-11-19 22:02:23 +07:00 · 1315e836a4
parent b8ccce250a 2e4289e75c
commit 1315e836a4
17 changed files with 626 additions and 343 deletions
--- a/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
+++ b/.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml
@ -38,6 +38,6 @@ body:
    attributes:
      label: Relevant log output
      description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      render: Shell
    validations:
      required: true
--- a/tubearchivist/api/src/aggs.py
+++ b/tubearchivist/api/src/aggs.py
@ -24,62 +24,159 @@ class AggBase:
        raise NotImplementedError


-class Primary(AggBase):
-    """primary aggregation for total documents indexed"""
+class Video(AggBase):
+    """get video stats"""

-    name = "primary"
-    path = "ta_video,ta_channel,ta_playlist,ta_subtitle,ta_download/_search"
+    name = "video_stats"
+    path = "ta_video/_search"
    data = {
        "size": 0,
        "aggs": {
            "video_type": {
-                "filter": {"exists": {"field": "active"}},
-                "aggs": {"filtered": {"terms": {"field": "vid_type"}}},
+                "terms": {"field": "vid_type"},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
            },
-            "channel_total": {"value_count": {"field": "channel_active"}},
-            "channel_sub": {"terms": {"field": "channel_subscribed"}},
-            "playlist_total": {"value_count": {"field": "playlist_active"}},
-            "playlist_sub": {"terms": {"field": "playlist_subscribed"}},
-            "download": {"terms": {"field": "status"}},
+            "video_active": {
+                "terms": {"field": "active"},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
+            },
+            "video_media_size": {"sum": {"field": "media_size"}},
+            "video_count": {"value_count": {"field": "youtube_id"}},
+            "duration": {"sum": {"field": "player.duration"}},
        },
    }

    def process(self):
-        """make the call"""
+        """process aggregation"""
        aggregations = self.get()

-        videos = {"total": aggregations["video_type"].get("doc_count")}
-        videos.update(
-            {
-                i.get("key"): i.get("doc_count")
-                for i in aggregations["video_type"]["filtered"]["buckets"]
-            }
-        )
-        channels = {"total": aggregations["channel_total"].get("value")}
-        channels.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["channel_sub"]["buckets"]
-            }
-        )
-        playlists = {"total": aggregations["playlist_total"].get("value")}
-        playlists.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["playlist_sub"]["buckets"]
-            }
-        )
-        downloads = {
-            i.get("key"): i.get("doc_count")
-            for i in aggregations["download"]["buckets"]
+        duration = int(aggregations["duration"]["value"])
+        response = {
+            "doc_count": aggregations["video_count"]["value"],
+            "media_size": int(aggregations["video_media_size"]["value"]),
+            "duration": duration,
+            "duration_str": get_duration_str(duration),
        }
+        for bucket in aggregations["video_type"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"type_{bucket['key']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        for bucket in aggregations["video_active"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"active_{bucket['key_as_string']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        return response
+
+
+class Channel(AggBase):
+    """get channel stats"""
+
+    name = "channel_stats"
+    path = "ta_channel/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "channel_count": {"value_count": {"field": "channel_id"}},
+            "channel_active": {"terms": {"field": "channel_active"}},
+            "channel_subscribed": {"terms": {"field": "channel_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()

        response = {
-            "videos": videos,
-            "channels": channels,
-            "playlists": playlists,
-            "downloads": downloads,
+            "doc_count": aggregations["channel_count"].get("value"),
        }
+        for bucket in aggregations["channel_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["channel_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Playlist(AggBase):
+    """get playlist stats"""
+
+    name = "playlist_stats"
+    path = "ta_playlist/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "playlist_count": {"value_count": {"field": "playlist_id"}},
+            "playlist_active": {"terms": {"field": "playlist_active"}},
+            "playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {"doc_count": aggregations["playlist_count"].get("value")}
+        for bucket in aggregations["playlist_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["playlist_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Download(AggBase):
+    """get downloads queue stats"""
+
+    name = "download_queue_stats"
+    path = "ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "status": {"terms": {"field": "status"}},
+            "video_type": {
+                "filter": {"term": {"status": "pending"}},
+                "aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
+            },
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {}
+        for bucket in aggregations["status"]["buckets"]:
+            response.update({bucket["key"]: bucket.get("doc_count")})
+
+        for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
+            key = f"pending_{bucket['key']}"
+            response.update({key: bucket.get("doc_count")})

        return response

--- a/tubearchivist/api/urls.py
+++ b/tubearchivist/api/urls.py
@ -152,9 +152,24 @@ urlpatterns = [
        name="api-notification",
    ),
    path(
-        "stats/primary/",
-        views.StatPrimaryView.as_view(),
-        name="api-stats-primary",
+        "stats/video/",
+        views.StatVideoView.as_view(),
+        name="api-stats-video",
+    ),
+    path(
+        "stats/channel/",
+        views.StatChannelView.as_view(),
+        name="api-stats-channel",
+    ),
+    path(
+        "stats/playlist/",
+        views.StatPlaylistView.as_view(),
+        name="api-stats-playlist",
+    ),
+    path(
+        "stats/download/",
+        views.StatDownloadView.as_view(),
+        name="api-stats-download",
    ),
    path(
        "stats/watch/",
--- a/tubearchivist/api/views.py
+++ b/tubearchivist/api/views.py
@ -1,6 +1,14 @@
 """all API views"""

-from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress
+from api.src.aggs import (
+    BiggestChannel,
+    Channel,
+    Download,
+    DownloadHist,
+    Playlist,
+    Video,
+    WatchProgress,
+)
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
@ -1141,16 +1149,52 @@ class NotificationView(ApiBaseView):
        return Response(RedisArchivist().list_items(query))


-class StatPrimaryView(ApiBaseView):
-    """resolves to /api/stats/primary/
-    GET: return document count
+class StatVideoView(ApiBaseView):
+    """resolves to /api/stats/video/
+    GET: return video stats
    """

    def get(self, request):
        """get stats"""
        # pylint: disable=unused-argument

-        return Response(Primary().process())
+        return Response(Video().process())
+
+
+class StatChannelView(ApiBaseView):
+    """resolves to /api/stats/channel/
+    GET: return channel stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Channel().process())
+
+
+class StatPlaylistView(ApiBaseView):
+    """resolves to /api/stats/playlist/
+    GET: return playlist stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Playlist().process())
+
+
+class StatDownloadView(ApiBaseView):
+    """resolves to /api/stats/download/
+    GET: return download stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Download().process())


 class StatWatchProgress(ApiBaseView):
--- a/tubearchivist/home/src/es/index_mapping.json
+++ b/tubearchivist/home/src/es/index_mapping.json
@ -74,7 +74,7 @@
                            "type": "boolean"
                        },
                        "integrate_sponsorblock": {
-                            "type" : "boolean"
+                            "type": "boolean"
                        }
                    }
                }
@ -168,7 +168,7 @@
                                    "type": "boolean"
                                },
                                "integrate_sponsorblock": {
-                                    "type" : "boolean"
+                                    "type": "boolean"
                                }
                            }
                        }
@ -236,19 +236,37 @@
                "comment_count": {
                    "type": "long"
                },
-                "stats" : {
-                    "properties" : {
-                        "average_rating" : {
-                            "type" : "float"
+                "stats": {
+                    "properties": {
+                        "average_rating": {
+                            "type": "float"
                        },
-                        "dislike_count" : {
-                            "type" : "long"
+                        "dislike_count": {
+                            "type": "long"
                        },
-                        "like_count" : {
-                            "type" : "long"
+                        "like_count": {
+                            "type": "long"
                        },
-                        "view_count" : {
-                            "type" : "long"
+                        "view_count": {
+                            "type": "long"
+                        }
+                    }
+                },
+                "player": {
+                    "properties": {
+                        "duration": {
+                            "type": "long"
+                        },
+                        "duration_str": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "watched": {
+                            "type": "boolean"
+                        },
+                        "watched_date": {
+                            "type": "date",
+                            "format": "epoch_second"
                        }
                    }
                },
@ -314,28 +332,28 @@
                        "is_enabled": {
                            "type": "boolean"
                        },
-                        "segments" : {
-                            "properties" : {
-                                "UUID" : {
+                        "segments": {
+                            "properties": {
+                                "UUID": {
                                    "type": "keyword"
                                },
-                                "actionType" : {
+                                "actionType": {
                                    "type": "keyword"
                                },
-                                "category" : {
+                                "category": {
                                    "type": "keyword"
                                },
-                                "locked" : {
-                                    "type" : "short"
+                                "locked": {
+                                    "type": "short"
                                },
-                                "segment" : {
-                                    "type" : "float"
+                                "segment": {
+                                    "type": "float"
                                },
-                                "videoDuration" : {
-                                    "type" : "float"
+                                "videoDuration": {
+                                    "type": "float"
                                },
-                                "votes" : {
-                                    "type" : "long"
+                                "votes": {
+                                    "type": "long"
                                }
                            }
                        }
@ -516,7 +534,7 @@
                    "format": "epoch_second"
                },
                "subtitle_index": {
-                    "type" : "long"
+                    "type": "long"
                },
                "subtitle_lang": {
                    "type": "keyword"
@ -525,7 +543,7 @@
                    "type": "keyword"
                },
                "subtitle_line": {
-                    "type" : "text",
+                    "type": "text",
                    "analyzer": "english"
                }
            },
@ -560,14 +578,14 @@
                            "type": "keyword"
                        },
                        "comment_text": {
-                            "type" : "text"
+                            "type": "text"
                        },
                        "comment_timestamp": {
                            "type": "date",
                            "format": "epoch_second"
                        },
                        "comment_time_text": {
-                            "type" : "text"
+                            "type": "text"
                        },
                        "comment_likecount": {
                            "type": "long"
@ -613,4 +631,4 @@
            }
        }
    ]
-}
+}
--- a/tubearchivist/home/src/index/comments.py
+++ b/tubearchivist/home/src/index/comments.py
@ -77,7 +77,7 @@ class Comments:
    def get_yt_comments(self):
        """get comments from youtube"""
        yt_obs = self.build_yt_obs()
-        info_json = YtWrap(yt_obs).extract(self.youtube_id)
+        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
        if not info_json:
            return False, False

--- a/tubearchivist/home/src/index/reindex.py
+++ b/tubearchivist/home/src/index/reindex.py
@ -105,11 +105,13 @@ class ReindexPopulate(ReindexBase):
        """get total hits from index"""
        index_name = reindex_config["index_name"]
        active_key = reindex_config["active_key"]
-        path = f"{index_name}/_search?filter_path=hits.total"
-        data = {"query": {"match": {active_key: True}}}
-        response, _ = ElasticWrap(path).post(data=data)
-        total_hits = response["hits"]["total"]["value"]
-        return total_hits
+        data = {
+            "query": {"term": {active_key: {"value": True}}},
+            "_source": False,
+        }
+        total = IndexPaginate(index_name, data, keep_source=True).get_results()
+
+        return len(total)

    def _get_daily_should(self, total_hits):
        """calc how many should reindex daily"""
@ -123,7 +125,7 @@ class ReindexPopulate(ReindexBase):
        """get outdated from index_name"""
        index_name = reindex_config["index_name"]
        refresh_key = reindex_config["refresh_key"]
-        now_lte = self.now - self.interval * 24 * 60 * 60
+        now_lte = str(self.now - self.interval * 24 * 60 * 60)
        must_list = [
            {"match": {reindex_config["active_key"]: True}},
            {"range": {refresh_key: {"lte": now_lte}}},
--- a/tubearchivist/home/src/index/video.py
+++ b/tubearchivist/home/src/index/video.py
@ -188,11 +188,11 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
        # build json_data basics
        self.json_data = {
            "title": self.youtube_meta["title"],
-            "description": self.youtube_meta["description"],
-            "category": self.youtube_meta["categories"],
+            "description": self.youtube_meta.get("description", ""),
+            "category": self.youtube_meta.get("categories", []),
            "vid_thumb_url": self.youtube_meta["thumbnail"],
            "vid_thumb_base64": base64_blur,
-            "tags": self.youtube_meta["tags"],
+            "tags": self.youtube_meta.get("tags", []),
            "published": published,
            "vid_last_refresh": last_refresh,
            "date_downloaded": last_refresh,
@ -210,20 +210,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):

    def _add_stats(self):
        """add stats dicst to json_data"""
-        # likes
-        like_count = self.youtube_meta.get("like_count", 0)
-        dislike_count = self.youtube_meta.get("dislike_count", 0)
-        average_rating = self.youtube_meta.get("average_rating", 0)
-        self.json_data.update(
-            {
-                "stats": {
-                    "view_count": self.youtube_meta["view_count"],
-                    "like_count": like_count,
-                    "dislike_count": dislike_count,
-                    "average_rating": average_rating,
-                }
-            }
-        )
+        stats = {
+            "view_count": self.youtube_meta.get("view_count", 0),
+            "like_count": self.youtube_meta.get("like_count", 0),
+            "dislike_count": self.youtube_meta.get("dislike_count", 0),
+            "average_rating": self.youtube_meta.get("average_rating", 0),
+        }
+        self.json_data.update({"stats": stats})

    def build_dl_cache_path(self):
        """find video path in dl cache"""
--- a/tubearchivist/home/src/ta/users.py
+++ b/tubearchivist/home/src/ta/users.py
@ -50,7 +50,14 @@ class UserConfig:
    VALID_STYLESHEETS = get_stylesheets()
    VALID_VIEW_STYLE = ["grid", "list"]
    VALID_SORT_ORDER = ["asc", "desc"]
-    VALID_SORT_BY = ["published", "downloaded", "views", "likes"]
+    VALID_SORT_BY = [
+        "published",
+        "downloaded",
+        "views",
+        "likes",
+        "duration",
+        "filesize",
+    ]
    VALID_GRID_ITEMS = range(3, 8)

    def __init__(self, user_id: str):
--- a/tubearchivist/home/templates/home/channel_id.html
+++ b/tubearchivist/home/templates/home/channel_id.html
@ -82,6 +82,8 @@
                    <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                    <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                    <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                </select>
                <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                    <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
--- a/tubearchivist/home/templates/home/home.html
+++ b/tubearchivist/home/templates/home/home.html
@ -65,6 +65,8 @@
                    <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                    <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                    <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                </select>
                <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                    <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>
--- a/tubearchivist/home/templates/home/settings.html
+++ b/tubearchivist/home/templates/home/settings.html
@ -5,14 +5,26 @@
    <h1>Your Archive</h1>
 </div>
 <div class="settings-item">
-    <h2>Main overview</h2>
-    <div id="primaryBox" class="info-box info-box-4">
+    <h2>Overview</h2>
+    <div id="activeBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Video Type</h2>
+    <div id="videoTypeBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Application</h2>
+    <div id="secondaryBox" class="info-box info-box-3">
        <p id="loading">Loading...</p>
    </div>
 </div>
 <div class="settings-item">
    <h2>Watch Progress</h2>
-    <div id="watchBox" class="info-box info-box-3">
+    <div id="watchBox" class="info-box info-box-2">
        <p id="loading">Loading...</p>
    </div>
 </div>
--- a/tubearchivist/home/templates/home/settings_scheduling.html
+++ b/tubearchivist/home/templates/home/settings_scheduling.html
@ -51,7 +51,7 @@
        </div>
    </div>
    <div class="settings-group">
-        <h2>Start download</h2>
+        <h2>Start Download</h2>
        <div class="settings-item">
            <p>Current Download schedule: <span class="settings-current">
                {% if config.scheduler.download_pending %}
@ -112,7 +112,7 @@
        </div>
    </div>
    <div class="settings-group">
-        <h2>Thumbnail check</h2>
+        <h2>Thumbnail Check</h2>
        <div class="settings-item">
            <p>Current thumbnail check schedule: <span class="settings-current">
                {% if config.scheduler.thumbnail_check %}
--- a/tubearchivist/home/views.py
+++ b/tubearchivist/home/views.py
@ -111,6 +111,8 @@ class ArchivistResultsView(ArchivistViewConfig):
            "likes": "stats.like_count",
            "downloaded": "date_downloaded",
            "published": "published",
+            "duration": "player.duration",
+            "filesize": "media_size",
        }
        sort_by = sort_by_map[self.context["sort_by"]]

--- a/tubearchivist/requirements.txt
+++ b/tubearchivist/requirements.txt
@ -1,8 +1,8 @@
 apprise==1.6.0
-celery==5.3.4
+celery==5.3.5
 Django==4.2.7
 django-auth-ldap==4.6.0
-django-cors-headers==4.3.0
+django-cors-headers==4.3.1
 djangorestframework==3.14.0
 Pillow==10.1.0
 redis==5.0.1
@ -10,4 +10,4 @@ requests==2.31.0
 ryd-client==0.0.6
 uWSGI==2.0.23
 whitenoise==6.6.0
-yt-dlp==2023.10.13
+yt-dlp==2023.11.16
--- a/tubearchivist/static/css/style.css
+++ b/tubearchivist/static/css/style.css
@ -180,6 +180,10 @@ button:hover {
    text-decoration: underline;
 }

+.footer .boxed-content {
+    text-align: center;
+}
+
 /* toggle on-off */
 .toggle {
    display: flex;
--- a/tubearchivist/static/stats.js
+++ b/tubearchivist/static/stats.js
@ -5,29 +5,333 @@
 /* globals apiRequest */

 function primaryStats() {
-  let apiEndpoint = '/api/stats/primary/';
+  let apiVideoEndpoint = '/api/stats/video/';
+  let responseData = apiRequest(apiVideoEndpoint, 'GET');
+
+  let activeBox = document.getElementById('activeBox');
+  clearLoading(activeBox);
+
+  let totalTile = buildTotalVideoTile(responseData);
+  activeBox.appendChild(totalTile);
+  let activeTile = buildActiveVideoTile(responseData);
+  activeBox.appendChild(activeTile);
+  let inActiveTile = buildInActiveVideoTile(responseData);
+  activeBox.appendChild(inActiveTile);
+
+  let videoTypeBox = document.getElementById('videoTypeBox');
+  clearLoading(videoTypeBox);
+
+  let videosTypeTile = buildVideosTypeTile(responseData);
+  videoTypeBox.appendChild(videosTypeTile);
+  let shortsTypeTile = buildShortsTypeTile(responseData);
+  videoTypeBox.appendChild(shortsTypeTile);
+  let streamsTypeTile = buildStreamsTypeTile(responseData);
+  videoTypeBox.appendChild(streamsTypeTile);
+}
+
+function secondaryStats() {
+  let apiChannelEndpoint = '/api/stats/channel/';
+  let channelResponseData = apiRequest(apiChannelEndpoint, 'GET');
+  let secondaryBox = document.getElementById('secondaryBox');
+  clearLoading(secondaryBox);
+  let channelTile = buildChannelTile(channelResponseData);
+  secondaryBox.appendChild(channelTile);
+
+  let apiPlaylistEndpoint = '/api/stats/playlist/';
+  let playlistResponseData = apiRequest(apiPlaylistEndpoint, 'GET');
+  let playlistTile = buildPlaylistTile(playlistResponseData);
+  secondaryBox.appendChild(playlistTile);
+
+  let apiDownloadEndpoint = '/api/stats/download/';
+  let downloadResponseData = apiRequest(apiDownloadEndpoint, 'GET');
+  let downloadTile = buildDownloadTile(downloadResponseData);
+  secondaryBox.appendChild(downloadTile);
+}
+
+function buildTotalVideoTile(responseData) {
+  const totalCount = responseData.doc_count || 0;
+  const totalSize = humanFileSize(responseData.media_size || 0);
+  const content = {
+    Items: `${totalCount}`,
+    'Media Size': `${totalSize}`,
+    Duration: responseData.duration_str,
+  };
+  const tile = buildTile('All: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildActiveVideoTile(responseData) {
+  const activeCount = responseData?.active_true?.doc_count || 0;
+  const activeSize = humanFileSize(responseData?.active_true?.media_size || 0);
+  const duration = responseData?.active_true?.duration_str || 'NA';
+  const content = {
+    Items: `${activeCount}`,
+    'Media Size': `${activeSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Active: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildInActiveVideoTile(responseData) {
+  const inActiveCount = responseData?.active_false?.doc_count || 0;
+  const inActiveSize = humanFileSize(responseData?.active_false?.media_size || 0);
+  const duration = responseData?.active_false?.duration_str || 'NA';
+  const content = {
+    Items: `${inActiveCount}`,
+    'Media Size': `${inActiveSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Inactive: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildVideosTypeTile(responseData) {
+  const videosCount = responseData?.type_videos?.doc_count || 0;
+  const videosSize = humanFileSize(responseData?.type_videos?.media_size || 0);
+  const duration = responseData?.type_videos?.duration_str || 'NA';
+  const content = {
+    Items: `${videosCount}`,
+    'Media Size': `${videosSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Regular Videos: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildShortsTypeTile(responseData) {
+  const shortsCount = responseData?.type_shorts?.doc_count || 0;
+  const shortsSize = humanFileSize(responseData?.type_shorts?.media_size || 0);
+  const duration = responseData?.type_shorts?.duration_str || 'NA';
+  const content = {
+    Items: `${shortsCount}`,
+    'Media Size': `${shortsSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Shorts: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildStreamsTypeTile(responseData) {
+  const streamsCount = responseData?.type_streams?.doc_count || 0;
+  const streamsSize = humanFileSize(responseData?.type_streams?.media_size || 0);
+  const duration = responseData?.type_streams?.duration_str || 'NA';
+  const content = {
+    Items: `${streamsCount}`,
+    'Media Size': `${streamsSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Streams: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildChannelTile(responseData) {
+  let tile = buildTile('Channels: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
+  const content = {
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
+  };
+  const table = buildTileContenTable(content, 3);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function buildPlaylistTile(responseData) {
+  let tile = buildTile('Playlists: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
+  const content = {
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
+  };
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function buildDownloadTile(responseData) {
+  const pendingTotal = responseData.pending || 0;
+  let tile = buildTile(`Downloads Pending: ${pendingTotal}`);
+  const pendingVideos = responseData.pending_videos || 0;
+  const pendingShorts = responseData.pending_shorts || 0;
+  const pendingStreams = responseData.pending_streams || 0;
+  const content = {
+    Videos: pendingVideos,
+    Shorts: pendingShorts,
+    Streams: pendingStreams,
+  };
+  const table = buildTileContenTable(content, 3);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function watchStats() {
+  let apiEndpoint = '/api/stats/watch/';
  let responseData = apiRequest(apiEndpoint, 'GET');
-  let primaryBox = document.getElementById('primaryBox');
+  let watchBox = document.getElementById('watchBox');
+  clearLoading(watchBox);

-  clearLoading(primaryBox);
+  let watchedTile = buildWatchTile('watched', responseData.watched);
+  watchBox.appendChild(watchedTile);

-  let videoTile = buildVideoTile(responseData);
-  primaryBox.appendChild(videoTile);
+  let unwatchedTile = buildWatchTile('unwatched', responseData.unwatched);
+  watchBox.appendChild(unwatchedTile);
+}

-  let channelTile = buildChannelTile(responseData);
-  primaryBox.appendChild(channelTile);
+function buildWatchTile(title, watchDetail) {
+  const items = watchDetail?.items ?? 0;
+  const duration = watchDetail?.duration ?? 0;
+  const duration_str = watchDetail?.duration_str ?? '0s';
+  const hasProgess = !!watchDetail?.progress;
+  const progress = (Number(watchDetail?.progress) * 100).toFixed(2) ?? '0';

-  let playlistTile = buildPlaylistTile(responseData);
-  primaryBox.appendChild(playlistTile);
+  let titleCapizalized = capitalizeFirstLetter(title);

-  let downloadTile = buildDownloadTile(responseData);
-  primaryBox.appendChild(downloadTile);
+  if (hasProgess) {
+    titleCapizalized = `${progress}% ` + titleCapizalized;
+  }
+
+  let tile = buildTile(titleCapizalized);
+
+  const content = {
+    Videos: items,
+    Seconds: duration,
+    Duration: duration_str,
+  };
+
+  const table = buildTileContenTable(content, 3);
+
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function downloadHist() {
+  let apiEndpoint = '/api/stats/downloadhist/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let histBox = document.getElementById('downHistBox');
+  clearLoading(histBox);
+  if (responseData.length === 0) {
+    let tile = buildTile('No recent downloads');
+    histBox.appendChild(tile);
+    return;
+  }
+
+  for (let i = 0; i < responseData.length; i++) {
+    const dailyStat = responseData[i];
+    let tile = buildDailyStat(dailyStat);
+    histBox.appendChild(tile);
+  }
+}
+
+function buildDailyStat(dailyStat) {
+  let tile = buildTile(dailyStat.date);
+  let message = document.createElement('p');
+  const isExactlyOne = dailyStat.count === 1;
+
+  let text = 'Videos';
+  if (isExactlyOne) {
+    text = 'Video';
+  }
+
+  message.innerText = `+${dailyStat.count} ${text}
+  ${humanFileSize(dailyStat.media_size)}`;
+
+  tile.appendChild(message);
+  return tile;
+}
+
+function buildChannelRow(id, name, value) {
+  let tableRow = document.createElement('tr');
+
+  tableRow.innerHTML = `
+    <td class="agg-channel-name"><a href="/channel/${id}/">${name}</a></td>
+    <td class="agg-channel-right-align">${value}</td>
+  `;
+
+  return tableRow;
+}
+
+function addBiggestChannelByDocCount() {
+  let tBody = document.getElementById('biggestChannelTableVideos');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=doc_count';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, doc_count } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, doc_count);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByDuration() {
+  const tBody = document.getElementById('biggestChannelTableDuration');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=duration';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, duration_str } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, duration_str);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByMediaSize() {
+  let tBody = document.getElementById('biggestChannelTableMediaSize');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=media_size';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, media_size } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, humanFileSize(media_size));
+
+    tBody.appendChild(tableRow);
+  }
 }

 function clearLoading(dashBox) {
  dashBox.querySelector('#loading').remove();
 }

+function capitalizeFirstLetter(string) {
+  // source: https://stackoverflow.com/a/1026087
+  return string.charAt(0).toUpperCase() + string.slice(1);
+}
+
+function humanFileSize(size) {
+  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
+  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
 function buildTile(titleText) {
  let tile = document.createElement('div');
  tile.classList.add('info-box-item');
@ -89,226 +393,6 @@ function buildTileContenTable(content, rowsWanted) {
  return table;
 }

-function buildVideoTile(responseData) {
-  let tile = buildTile(`Video types: `);
-
-  const total = responseData.videos.total || 0;
-  const videos = responseData.videos.videos || 0;
-  const shorts = responseData.videos.shorts || 0;
-  const streams = responseData.videos.streams || 0;
-
-  const content = {
-    Videos: `${videos}/${total}`,
-    Shorts: `${shorts}/${total}`,
-    Streams: `${streams}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildChannelTile(responseData) {
-  let tile = buildTile(`Channels: `);
-
-  const total = responseData.channels.total || 0;
-  const subscribed = responseData.channels.sub_true || 0;
-
-  const content = {
-    Subscribed: `${subscribed}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildPlaylistTile(responseData) {
-  let tile = buildTile(`Playlists:`);
-
-  const total = responseData.playlists.total || 0;
-  const subscribed = responseData.playlists.sub_true || 0;
-
-  const content = {
-    Subscribed: `${subscribed}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildDownloadTile(responseData) {
-  let tile = buildTile('Downloads');
-
-  const pending = responseData.downloads.pending || 0;
-  const ignored = responseData.downloads.ignore || 0;
-
-  const content = {
-    Pending: pending,
-    Ignored: ignored,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function watchStats() {
-  let apiEndpoint = '/api/stats/watch/';
-  let responseData = apiRequest(apiEndpoint, 'GET');
-  let watchBox = document.getElementById('watchBox');
-  clearLoading(watchBox);
-
-  const { total, watched, unwatched } = responseData;
-
-  let firstCard = buildWatchTile('total', total);
-  watchBox.appendChild(firstCard);
-
-  let secondCard = buildWatchTile('watched', watched);
-  watchBox.appendChild(secondCard);
-
-  let thirdCard = buildWatchTile('unwatched', unwatched);
-  watchBox.appendChild(thirdCard);
-}
-
-function capitalizeFirstLetter(string) {
-  // source: https://stackoverflow.com/a/1026087
-  return string.charAt(0).toUpperCase() + string.slice(1);
-}
-
-function buildWatchTile(title, watchDetail) {
-  const items = watchDetail?.items ?? 0;
-  const duration = watchDetail?.duration ?? 0;
-  const duration_str = watchDetail?.duration_str ?? '0s';
-  const hasProgess = !!watchDetail?.progress;
-  const progress = (Number(watchDetail?.progress) * 100).toFixed(2) ?? '0';
-
-  let titleCapizalized = capitalizeFirstLetter(title);
-
-  if (hasProgess) {
-    titleCapizalized = `${progress}% ` + titleCapizalized;
-  }
-
-  let tile = buildTile(titleCapizalized);
-
-  const content = {
-    Videos: items,
-    Seconds: duration,
-    Playback: duration_str,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function downloadHist() {
-  let apiEndpoint = '/api/stats/downloadhist/';
-  let responseData = apiRequest(apiEndpoint, 'GET');
-  let histBox = document.getElementById('downHistBox');
-  clearLoading(histBox);
-  if (responseData.length === 0) {
-    let tile = buildTile('No recent downloads');
-    histBox.appendChild(tile);
-    return;
-  }
-
-  for (let i = 0; i < responseData.length; i++) {
-    const dailyStat = responseData[i];
-    let tile = buildDailyStat(dailyStat);
-    histBox.appendChild(tile);
-  }
-}
-
-function buildDailyStat(dailyStat) {
-  let tile = buildTile(dailyStat.date);
-  let message = document.createElement('p');
-  const isExactlyOne = dailyStat.count === 1;
-
-  let text = 'Videos';
-  if (isExactlyOne) {
-    text = 'Video';
-  }
-
-  message.innerText = `+${dailyStat.count} ${text}
-  ${humanFileSize(dailyStat.media_size)}`;
-
-  tile.appendChild(message);
-  return tile;
-}
-
-function humanFileSize(size) {
-  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
-  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
-}
-
-function buildChannelRow(id, name, value) {
-  let tableRow = document.createElement('tr');
-
-  tableRow.innerHTML = `
-    <td class="agg-channel-name"><a href="/channel/${id}/">${name}</a></td>
-    <td class="agg-channel-right-align">${value}</td>
-  `;
-
-  return tableRow;
-}
-
-function addBiggestChannelByDocCount() {
-  let tBody = document.getElementById('biggestChannelTableVideos');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=doc_count';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, doc_count } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, doc_count);
-
-    tBody.appendChild(tableRow);
-  }
-}
-
-function addBiggestChannelByDuration() {
-  const tBody = document.getElementById('biggestChannelTableDuration');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=duration';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, duration_str } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, duration_str);
-
-    tBody.appendChild(tableRow);
-  }
-}
-
-function addBiggestChannelByMediaSize() {
-  let tBody = document.getElementById('biggestChannelTableMediaSize');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=media_size';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, media_size } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, humanFileSize(media_size));
-
-    tBody.appendChild(tableRow);
-  }
-}
-
 function biggestChannel() {
  addBiggestChannelByDocCount();
  addBiggestChannelByDuration();
@ -317,6 +401,7 @@ function biggestChannel() {

 async function buildStats() {
  primaryStats();
+  secondaryStats();
  watchStats();
  downloadHist();
  biggestChannel();