Improved dashboard, reindex fix, #build

Changed:
- Added additional sort by fields
- [API] Chaned primary stats endpoints
- [API] Added separate video stats endpoints
- Added fallback for some manual import values
- Fix comment extration for members video
- Fix reindex outdated query

.github/ISSUE_TEMPLATE/INSTALLATION-HELP.yml

@@ -38,6 +38,6 @@ body:
     attributes:
       label: Relevant log output
       description: Please copy and paste any relevant Docker logs. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      render: Shell
     validations:
       required: true

tubearchivist/api/src/aggs.py

@@ -24,62 +24,159 @@ class AggBase:
         raise NotImplementedError
 
 
-class Primary(AggBase):
+class Video(AggBase):
-    """primary aggregation for total documents indexed"""
+    """get video stats"""
 
-    name = "primary"
+    name = "video_stats"
-    path = "ta_video,ta_channel,ta_playlist,ta_subtitle,ta_download/_search"
+    path = "ta_video/_search"
     data = {
         "size": 0,
         "aggs": {
             "video_type": {
-                "filter": {"exists": {"field": "active"}},
+                "terms": {"field": "vid_type"},
-                "aggs": {"filtered": {"terms": {"field": "vid_type"}}},
+                "aggs": {
+                    "media_size": {"sum": {"field": "media_size"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
             },
-            "channel_total": {"value_count": {"field": "channel_active"}},
+            "video_active": {
-            "channel_sub": {"terms": {"field": "channel_subscribed"}},
+                "terms": {"field": "active"},
-            "playlist_total": {"value_count": {"field": "playlist_active"}},
+                "aggs": {
-            "playlist_sub": {"terms": {"field": "playlist_subscribed"}},
+                    "media_size": {"sum": {"field": "media_size"}},
-            "download": {"terms": {"field": "status"}},
+                    "duration": {"sum": {"field": "player.duration"}},
+                },
+            },
+            "video_media_size": {"sum": {"field": "media_size"}},
+            "video_count": {"value_count": {"field": "youtube_id"}},
+            "duration": {"sum": {"field": "player.duration"}},
         },
     }
 
     def process(self):
-        """make the call"""
+        """process aggregation"""
         aggregations = self.get()
 
-        videos = {"total": aggregations["video_type"].get("doc_count")}
+        duration = int(aggregations["duration"]["value"])
-        videos.update(
+        response = {
-            {
+            "doc_count": aggregations["video_count"]["value"],
-                i.get("key"): i.get("doc_count")
+            "media_size": int(aggregations["video_media_size"]["value"]),
-                for i in aggregations["video_type"]["filtered"]["buckets"]
+            "duration": duration,
-            }
+            "duration_str": get_duration_str(duration),
-        )
-        channels = {"total": aggregations["channel_total"].get("value")}
-        channels.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["channel_sub"]["buckets"]
-            }
-        )
-        playlists = {"total": aggregations["playlist_total"].get("value")}
-        playlists.update(
-            {
-                "sub_" + i.get("key_as_string"): i.get("doc_count")
-                for i in aggregations["playlist_sub"]["buckets"]
-            }
-        )
-        downloads = {
-            i.get("key"): i.get("doc_count")
-            for i in aggregations["download"]["buckets"]
         }
+        for bucket in aggregations["video_type"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"type_{bucket['key']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        for bucket in aggregations["video_active"]["buckets"]:
+            duration = int(bucket["duration"].get("value"))
+            response.update(
+                {
+                    f"active_{bucket['key_as_string']}": {
+                        "doc_count": bucket.get("doc_count"),
+                        "media_size": int(bucket["media_size"].get("value")),
+                        "duration": duration,
+                        "duration_str": get_duration_str(duration),
+                    }
+                }
+            )
+
+        return response
+
+
+class Channel(AggBase):
+    """get channel stats"""
+
+    name = "channel_stats"
+    path = "ta_channel/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "channel_count": {"value_count": {"field": "channel_id"}},
+            "channel_active": {"terms": {"field": "channel_active"}},
+            "channel_subscribed": {"terms": {"field": "channel_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
 
         response = {
-            "videos": videos,
+            "doc_count": aggregations["channel_count"].get("value"),
-            "channels": channels,
-            "playlists": playlists,
-            "downloads": downloads,
         }
+        for bucket in aggregations["channel_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["channel_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Playlist(AggBase):
+    """get playlist stats"""
+
+    name = "playlist_stats"
+    path = "ta_playlist/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "playlist_count": {"value_count": {"field": "playlist_id"}},
+            "playlist_active": {"terms": {"field": "playlist_active"}},
+            "playlist_subscribed": {"terms": {"field": "playlist_subscribed"}},
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {"doc_count": aggregations["playlist_count"].get("value")}
+        for bucket in aggregations["playlist_active"]["buckets"]:
+            key = f"active_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+        for bucket in aggregations["playlist_subscribed"]["buckets"]:
+            key = f"subscribed_{bucket['key_as_string']}"
+            response.update({key: bucket.get("doc_count")})
+
+        return response
+
+
+class Download(AggBase):
+    """get downloads queue stats"""
+
+    name = "download_queue_stats"
+    path = "ta_download/_search"
+    data = {
+        "size": 0,
+        "aggs": {
+            "status": {"terms": {"field": "status"}},
+            "video_type": {
+                "filter": {"term": {"status": "pending"}},
+                "aggs": {"type_pending": {"terms": {"field": "vid_type"}}},
+            },
+        },
+    }
+
+    def process(self):
+        """process aggregation"""
+        aggregations = self.get()
+        response = {}
+        for bucket in aggregations["status"]["buckets"]:
+            response.update({bucket["key"]: bucket.get("doc_count")})
+
+        for bucket in aggregations["video_type"]["type_pending"]["buckets"]:
+            key = f"pending_{bucket['key']}"
+            response.update({key: bucket.get("doc_count")})
 
         return response
 

tubearchivist/api/urls.py

@@ -152,9 +152,24 @@ urlpatterns = [
         name="api-notification",
     ),
     path(
-        "stats/primary/",
+        "stats/video/",
-        views.StatPrimaryView.as_view(),
+        views.StatVideoView.as_view(),
-        name="api-stats-primary",
+        name="api-stats-video",
+    ),
+    path(
+        "stats/channel/",
+        views.StatChannelView.as_view(),
+        name="api-stats-channel",
+    ),
+    path(
+        "stats/playlist/",
+        views.StatPlaylistView.as_view(),
+        name="api-stats-playlist",
+    ),
+    path(
+        "stats/download/",
+        views.StatDownloadView.as_view(),
+        name="api-stats-download",
     ),
     path(
         "stats/watch/",

tubearchivist/api/views.py

@@ -1,6 +1,14 @@
 """all API views"""
 
-from api.src.aggs import BiggestChannel, DownloadHist, Primary, WatchProgress
+from api.src.aggs import (
+    BiggestChannel,
+    Channel,
+    Download,
+    DownloadHist,
+    Playlist,
+    Video,
+    WatchProgress,
+)
 from api.src.search_processor import SearchProcess
 from home.src.download.queue import PendingInteract
 from home.src.download.subscriptions import (
@@ -1141,16 +1149,52 @@ class NotificationView(ApiBaseView):
         return Response(RedisArchivist().list_items(query))
 
 
-class StatPrimaryView(ApiBaseView):
+class StatVideoView(ApiBaseView):
-    """resolves to /api/stats/primary/
+    """resolves to /api/stats/video/
-    GET: return document count
+    GET: return video stats
     """
 
     def get(self, request):
         """get stats"""
         # pylint: disable=unused-argument
 
-        return Response(Primary().process())
+        return Response(Video().process())
+
+
+class StatChannelView(ApiBaseView):
+    """resolves to /api/stats/channel/
+    GET: return channel stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Channel().process())
+
+
+class StatPlaylistView(ApiBaseView):
+    """resolves to /api/stats/playlist/
+    GET: return playlist stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Playlist().process())
+
+
+class StatDownloadView(ApiBaseView):
+    """resolves to /api/stats/download/
+    GET: return download stats
+    """
+
+    def get(self, request):
+        """get stats"""
+        # pylint: disable=unused-argument
+
+        return Response(Download().process())
 
 
 class StatWatchProgress(ApiBaseView):

tubearchivist/home/src/es/index_mapping.json

@@ -74,7 +74,7 @@
                             "type": "boolean"
                         },
                         "integrate_sponsorblock": {
-                            "type" : "boolean"
+                            "type": "boolean"
                         }
                     }
                 }
@@ -168,7 +168,7 @@
                                     "type": "boolean"
                                 },
                                 "integrate_sponsorblock": {
-                                    "type" : "boolean"
+                                    "type": "boolean"
                                 }
                             }
                         }
@@ -236,19 +236,37 @@
                 "comment_count": {
                     "type": "long"
                 },
-                "stats" : {
+                "stats": {
-                    "properties" : {
+                    "properties": {
-                        "average_rating" : {
+                        "average_rating": {
-                            "type" : "float"
+                            "type": "float"
                         },
-                        "dislike_count" : {
+                        "dislike_count": {
-                            "type" : "long"
+                            "type": "long"
                         },
-                        "like_count" : {
+                        "like_count": {
-                            "type" : "long"
+                            "type": "long"
                         },
-                        "view_count" : {
+                        "view_count": {
-                            "type" : "long"
+                            "type": "long"
+                        }
+                    }
+                },
+                "player": {
+                    "properties": {
+                        "duration": {
+                            "type": "long"
+                        },
+                        "duration_str": {
+                            "type": "keyword",
+                            "index": false
+                        },
+                        "watched": {
+                            "type": "boolean"
+                        },
+                        "watched_date": {
+                            "type": "date",
+                            "format": "epoch_second"
                         }
                     }
                 },
@@ -314,28 +332,28 @@
                         "is_enabled": {
                             "type": "boolean"
                         },
-                        "segments" : {
+                        "segments": {
-                            "properties" : {
+                            "properties": {
-                                "UUID" : {
+                                "UUID": {
                                     "type": "keyword"
                                 },
-                                "actionType" : {
+                                "actionType": {
                                     "type": "keyword"
                                 },
-                                "category" : {
+                                "category": {
                                     "type": "keyword"
                                 },
-                                "locked" : {
+                                "locked": {
-                                    "type" : "short"
+                                    "type": "short"
                                 },
-                                "segment" : {
+                                "segment": {
-                                    "type" : "float"
+                                    "type": "float"
                                 },
-                                "videoDuration" : {
+                                "videoDuration": {
-                                    "type" : "float"
+                                    "type": "float"
                                 },
-                                "votes" : {
+                                "votes": {
-                                    "type" : "long"
+                                    "type": "long"
                                 }
                             }
                         }
@@ -516,7 +534,7 @@
                     "format": "epoch_second"
                 },
                 "subtitle_index": {
-                    "type" : "long"
+                    "type": "long"
                 },
                 "subtitle_lang": {
                     "type": "keyword"
@@ -525,7 +543,7 @@
                     "type": "keyword"
                 },
                 "subtitle_line": {
-                    "type" : "text",
+                    "type": "text",
                     "analyzer": "english"
                 }
             },
@@ -560,14 +578,14 @@
                             "type": "keyword"
                         },
                         "comment_text": {
-                            "type" : "text"
+                            "type": "text"
                         },
                         "comment_timestamp": {
                             "type": "date",
                             "format": "epoch_second"
                         },
                         "comment_time_text": {
-                            "type" : "text"
+                            "type": "text"
                         },
                         "comment_likecount": {
                             "type": "long"
@@ -613,4 +631,4 @@
             }
         }
     ]
-}
+}

tubearchivist/home/src/index/comments.py

@@ -77,7 +77,7 @@ class Comments:
     def get_yt_comments(self):
         """get comments from youtube"""
         yt_obs = self.build_yt_obs()
-        info_json = YtWrap(yt_obs).extract(self.youtube_id)
+        info_json = YtWrap(yt_obs, config=self.config).extract(self.youtube_id)
         if not info_json:
             return False, False
 

tubearchivist/home/src/index/reindex.py

@@ -105,11 +105,13 @@ class ReindexPopulate(ReindexBase):
         """get total hits from index"""
         index_name = reindex_config["index_name"]
         active_key = reindex_config["active_key"]
-        path = f"{index_name}/_search?filter_path=hits.total"
+        data = {
-        data = {"query": {"match": {active_key: True}}}
+            "query": {"term": {active_key: {"value": True}}},
-        response, _ = ElasticWrap(path).post(data=data)
+            "_source": False,
-        total_hits = response["hits"]["total"]["value"]
+        }
-        return total_hits
+        total = IndexPaginate(index_name, data, keep_source=True).get_results()
+
+        return len(total)
 
     def _get_daily_should(self, total_hits):
         """calc how many should reindex daily"""
@@ -123,7 +125,7 @@ class ReindexPopulate(ReindexBase):
         """get outdated from index_name"""
         index_name = reindex_config["index_name"]
         refresh_key = reindex_config["refresh_key"]
-        now_lte = self.now - self.interval * 24 * 60 * 60
+        now_lte = str(self.now - self.interval * 24 * 60 * 60)
         must_list = [
             {"match": {reindex_config["active_key"]: True}},
             {"range": {refresh_key: {"lte": now_lte}}},

tubearchivist/home/src/index/video.py

@@ -188,11 +188,11 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
         # build json_data basics
         self.json_data = {
             "title": self.youtube_meta["title"],
-            "description": self.youtube_meta["description"],
+            "description": self.youtube_meta.get("description", ""),
-            "category": self.youtube_meta["categories"],
+            "category": self.youtube_meta.get("categories", []),
             "vid_thumb_url": self.youtube_meta["thumbnail"],
             "vid_thumb_base64": base64_blur,
-            "tags": self.youtube_meta["tags"],
+            "tags": self.youtube_meta.get("tags", []),
             "published": published,
             "vid_last_refresh": last_refresh,
             "date_downloaded": last_refresh,
@@ -210,20 +210,13 @@ class YoutubeVideo(YouTubeItem, YoutubeSubtitle):
 
     def _add_stats(self):
         """add stats dicst to json_data"""
-        # likes
+        stats = {
-        like_count = self.youtube_meta.get("like_count", 0)
+            "view_count": self.youtube_meta.get("view_count", 0),
-        dislike_count = self.youtube_meta.get("dislike_count", 0)
+            "like_count": self.youtube_meta.get("like_count", 0),
-        average_rating = self.youtube_meta.get("average_rating", 0)
+            "dislike_count": self.youtube_meta.get("dislike_count", 0),
-        self.json_data.update(
+            "average_rating": self.youtube_meta.get("average_rating", 0),
-            {
+        }
-                "stats": {
+        self.json_data.update({"stats": stats})
-                    "view_count": self.youtube_meta["view_count"],
-                    "like_count": like_count,
-                    "dislike_count": dislike_count,
-                    "average_rating": average_rating,
-                }
-            }
-        )
 
     def build_dl_cache_path(self):
         """find video path in dl cache"""

tubearchivist/home/src/ta/users.py

@@ -50,7 +50,14 @@ class UserConfig:
     VALID_STYLESHEETS = get_stylesheets()
     VALID_VIEW_STYLE = ["grid", "list"]
     VALID_SORT_ORDER = ["asc", "desc"]
-    VALID_SORT_BY = ["published", "downloaded", "views", "likes"]
+    VALID_SORT_BY = [
+        "published",
+        "downloaded",
+        "views",
+        "likes",
+        "duration",
+        "filesize",
+    ]
     VALID_GRID_ITEMS = range(3, 8)
 
     def __init__(self, user_id: str):

tubearchivist/home/templates/home/channel_id.html

@@ -82,6 +82,8 @@
                     <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                     <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                     <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                 </select>
                 <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                     <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>

tubearchivist/home/templates/home/home.html

@@ -65,6 +65,8 @@
                     <option value="downloaded" {% if sort_by == "downloaded" %}selected{% endif %}>date downloaded</option>
                     <option value="views" {% if sort_by == "views" %}selected{% endif %}>views</option>
                     <option value="likes" {% if sort_by == "likes" %}selected{% endif %}>likes</option>
+                    <option value="duration" {% if sort_by == "duration" %}selected{% endif %}>duration</option>
+                    <option value="filesize" {% if sort_by == "filesize" %}selected{% endif %}>file size</option>
                 </select>
                 <select name="sort_order" id="sort-order" onchange="sortChange(this)">
                     <option value="asc" {% if sort_order == "asc" %}selected{% endif %}>asc</option>

tubearchivist/home/templates/home/settings.html

@@ -5,14 +5,26 @@
     <h1>Your Archive</h1>
 </div>
 <div class="settings-item">
-    <h2>Main overview</h2>
+    <h2>Overview</h2>
-    <div id="primaryBox" class="info-box info-box-4">
+    <div id="activeBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Video Type</h2>
+    <div id="videoTypeBox" class="info-box info-box-3">
+        <p id="loading">Loading...</p>
+    </div>
+</div>
+<div class="settings-item">
+    <h2>Application</h2>
+    <div id="secondaryBox" class="info-box info-box-3">
         <p id="loading">Loading...</p>
     </div>
 </div>
 <div class="settings-item">
     <h2>Watch Progress</h2>
-    <div id="watchBox" class="info-box info-box-3">
+    <div id="watchBox" class="info-box info-box-2">
         <p id="loading">Loading...</p>
     </div>
 </div>

tubearchivist/home/templates/home/settings_scheduling.html

@@ -51,7 +51,7 @@
         </div>
     </div>
     <div class="settings-group">
-        <h2>Start download</h2>
+        <h2>Start Download</h2>
         <div class="settings-item">
             <p>Current Download schedule: <span class="settings-current">
                 {% if config.scheduler.download_pending %}
@@ -112,7 +112,7 @@
         </div>
     </div>
     <div class="settings-group">
-        <h2>Thumbnail check</h2>
+        <h2>Thumbnail Check</h2>
         <div class="settings-item">
             <p>Current thumbnail check schedule: <span class="settings-current">
                 {% if config.scheduler.thumbnail_check %}

tubearchivist/home/views.py

@@ -111,6 +111,8 @@ class ArchivistResultsView(ArchivistViewConfig):
             "likes": "stats.like_count",
             "downloaded": "date_downloaded",
             "published": "published",
+            "duration": "player.duration",
+            "filesize": "media_size",
         }
         sort_by = sort_by_map[self.context["sort_by"]]
 

tubearchivist/requirements.txt

@@ -1,8 +1,8 @@
 apprise==1.6.0
-celery==5.3.4
+celery==5.3.5
 Django==4.2.7
 django-auth-ldap==4.6.0
-django-cors-headers==4.3.0
+django-cors-headers==4.3.1
 djangorestframework==3.14.0
 Pillow==10.1.0
 redis==5.0.1
@@ -10,4 +10,4 @@ requests==2.31.0
 ryd-client==0.0.6
 uWSGI==2.0.23
 whitenoise==6.6.0
-yt-dlp==2023.10.13
+yt-dlp==2023.11.16

tubearchivist/static/css/style.css

@@ -180,6 +180,10 @@ button:hover {
     text-decoration: underline;
 }
 
+.footer .boxed-content {
+    text-align: center;
+}
+
 /* toggle on-off */
 .toggle {
     display: flex;

tubearchivist/static/stats.js

@@ -5,29 +5,333 @@
 /* globals apiRequest */
 
 function primaryStats() {
-  let apiEndpoint = '/api/stats/primary/';
+  let apiVideoEndpoint = '/api/stats/video/';
+  let responseData = apiRequest(apiVideoEndpoint, 'GET');
+
+  let activeBox = document.getElementById('activeBox');
+  clearLoading(activeBox);
+
+  let totalTile = buildTotalVideoTile(responseData);
+  activeBox.appendChild(totalTile);
+  let activeTile = buildActiveVideoTile(responseData);
+  activeBox.appendChild(activeTile);
+  let inActiveTile = buildInActiveVideoTile(responseData);
+  activeBox.appendChild(inActiveTile);
+
+  let videoTypeBox = document.getElementById('videoTypeBox');
+  clearLoading(videoTypeBox);
+
+  let videosTypeTile = buildVideosTypeTile(responseData);
+  videoTypeBox.appendChild(videosTypeTile);
+  let shortsTypeTile = buildShortsTypeTile(responseData);
+  videoTypeBox.appendChild(shortsTypeTile);
+  let streamsTypeTile = buildStreamsTypeTile(responseData);
+  videoTypeBox.appendChild(streamsTypeTile);
+}
+
+function secondaryStats() {
+  let apiChannelEndpoint = '/api/stats/channel/';
+  let channelResponseData = apiRequest(apiChannelEndpoint, 'GET');
+  let secondaryBox = document.getElementById('secondaryBox');
+  clearLoading(secondaryBox);
+  let channelTile = buildChannelTile(channelResponseData);
+  secondaryBox.appendChild(channelTile);
+
+  let apiPlaylistEndpoint = '/api/stats/playlist/';
+  let playlistResponseData = apiRequest(apiPlaylistEndpoint, 'GET');
+  let playlistTile = buildPlaylistTile(playlistResponseData);
+  secondaryBox.appendChild(playlistTile);
+
+  let apiDownloadEndpoint = '/api/stats/download/';
+  let downloadResponseData = apiRequest(apiDownloadEndpoint, 'GET');
+  let downloadTile = buildDownloadTile(downloadResponseData);
+  secondaryBox.appendChild(downloadTile);
+}
+
+function buildTotalVideoTile(responseData) {
+  const totalCount = responseData.doc_count || 0;
+  const totalSize = humanFileSize(responseData.media_size || 0);
+  const content = {
+    Items: `${totalCount}`,
+    'Media Size': `${totalSize}`,
+    Duration: responseData.duration_str,
+  };
+  const tile = buildTile('All: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildActiveVideoTile(responseData) {
+  const activeCount = responseData?.active_true?.doc_count || 0;
+  const activeSize = humanFileSize(responseData?.active_true?.media_size || 0);
+  const duration = responseData?.active_true?.duration_str || 'NA';
+  const content = {
+    Items: `${activeCount}`,
+    'Media Size': `${activeSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Active: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildInActiveVideoTile(responseData) {
+  const inActiveCount = responseData?.active_false?.doc_count || 0;
+  const inActiveSize = humanFileSize(responseData?.active_false?.media_size || 0);
+  const duration = responseData?.active_false?.duration_str || 'NA';
+  const content = {
+    Items: `${inActiveCount}`,
+    'Media Size': `${inActiveSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Inactive: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildVideosTypeTile(responseData) {
+  const videosCount = responseData?.type_videos?.doc_count || 0;
+  const videosSize = humanFileSize(responseData?.type_videos?.media_size || 0);
+  const duration = responseData?.type_videos?.duration_str || 'NA';
+  const content = {
+    Items: `${videosCount}`,
+    'Media Size': `${videosSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Regular Videos: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildShortsTypeTile(responseData) {
+  const shortsCount = responseData?.type_shorts?.doc_count || 0;
+  const shortsSize = humanFileSize(responseData?.type_shorts?.media_size || 0);
+  const duration = responseData?.type_shorts?.duration_str || 'NA';
+  const content = {
+    Items: `${shortsCount}`,
+    'Media Size': `${shortsSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Shorts: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildStreamsTypeTile(responseData) {
+  const streamsCount = responseData?.type_streams?.doc_count || 0;
+  const streamsSize = humanFileSize(responseData?.type_streams?.media_size || 0);
+  const duration = responseData?.type_streams?.duration_str || 'NA';
+  const content = {
+    Items: `${streamsCount}`,
+    'Media Size': `${streamsSize}`,
+    Duration: duration,
+  };
+  const tile = buildTile('Streams: ');
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+  return tile;
+}
+
+function buildChannelTile(responseData) {
+  let tile = buildTile('Channels: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
+  const content = {
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
+  };
+  const table = buildTileContenTable(content, 3);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function buildPlaylistTile(responseData) {
+  let tile = buildTile('Playlists: ');
+  const total = responseData.doc_count || 0;
+  const subscribed = responseData.subscribed_true || 0;
+  const active = responseData.active_true || 0;
+  const content = {
+    Subscribed: subscribed,
+    Active: active,
+    Total: total,
+  };
+  const table = buildTileContenTable(content, 2);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function buildDownloadTile(responseData) {
+  const pendingTotal = responseData.pending || 0;
+  let tile = buildTile(`Downloads Pending: ${pendingTotal}`);
+  const pendingVideos = responseData.pending_videos || 0;
+  const pendingShorts = responseData.pending_shorts || 0;
+  const pendingStreams = responseData.pending_streams || 0;
+  const content = {
+    Videos: pendingVideos,
+    Shorts: pendingShorts,
+    Streams: pendingStreams,
+  };
+  const table = buildTileContenTable(content, 3);
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function watchStats() {
+  let apiEndpoint = '/api/stats/watch/';
   let responseData = apiRequest(apiEndpoint, 'GET');
-  let primaryBox = document.getElementById('primaryBox');
+  let watchBox = document.getElementById('watchBox');
+  clearLoading(watchBox);
 
-  clearLoading(primaryBox);
+  let watchedTile = buildWatchTile('watched', responseData.watched);
+  watchBox.appendChild(watchedTile);
 
-  let videoTile = buildVideoTile(responseData);
+  let unwatchedTile = buildWatchTile('unwatched', responseData.unwatched);
-  primaryBox.appendChild(videoTile);
+  watchBox.appendChild(unwatchedTile);
+}
 
-  let channelTile = buildChannelTile(responseData);
+function buildWatchTile(title, watchDetail) {
-  primaryBox.appendChild(channelTile);
+  const items = watchDetail?.items ?? 0;
+  const duration = watchDetail?.duration ?? 0;
+  const duration_str = watchDetail?.duration_str ?? '0s';
+  const hasProgess = !!watchDetail?.progress;
+  const progress = (Number(watchDetail?.progress) * 100).toFixed(2) ?? '0';
 
-  let playlistTile = buildPlaylistTile(responseData);
+  let titleCapizalized = capitalizeFirstLetter(title);
-  primaryBox.appendChild(playlistTile);
 
-  let downloadTile = buildDownloadTile(responseData);
+  if (hasProgess) {
-  primaryBox.appendChild(downloadTile);
+    titleCapizalized = `${progress}% ` + titleCapizalized;
+  }
+
+  let tile = buildTile(titleCapizalized);
+
+  const content = {
+    Videos: items,
+    Seconds: duration,
+    Duration: duration_str,
+  };
+
+  const table = buildTileContenTable(content, 3);
+
+  tile.appendChild(table);
+
+  return tile;
+}
+
+function downloadHist() {
+  let apiEndpoint = '/api/stats/downloadhist/';
+  let responseData = apiRequest(apiEndpoint, 'GET');
+  let histBox = document.getElementById('downHistBox');
+  clearLoading(histBox);
+  if (responseData.length === 0) {
+    let tile = buildTile('No recent downloads');
+    histBox.appendChild(tile);
+    return;
+  }
+
+  for (let i = 0; i < responseData.length; i++) {
+    const dailyStat = responseData[i];
+    let tile = buildDailyStat(dailyStat);
+    histBox.appendChild(tile);
+  }
+}
+
+function buildDailyStat(dailyStat) {
+  let tile = buildTile(dailyStat.date);
+  let message = document.createElement('p');
+  const isExactlyOne = dailyStat.count === 1;
+
+  let text = 'Videos';
+  if (isExactlyOne) {
+    text = 'Video';
+  }
+
+  message.innerText = `+${dailyStat.count} ${text}
+  ${humanFileSize(dailyStat.media_size)}`;
+
+  tile.appendChild(message);
+  return tile;
+}
+
+function buildChannelRow(id, name, value) {
+  let tableRow = document.createElement('tr');
+
+  tableRow.innerHTML = `
+    <td class="agg-channel-name"><a href="/channel/${id}/">${name}</a></td>
+    <td class="agg-channel-right-align">${value}</td>
+  `;
+
+  return tableRow;
+}
+
+function addBiggestChannelByDocCount() {
+  let tBody = document.getElementById('biggestChannelTableVideos');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=doc_count';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, doc_count } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, doc_count);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByDuration() {
+  const tBody = document.getElementById('biggestChannelTableDuration');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=duration';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, duration_str } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, duration_str);
+
+    tBody.appendChild(tableRow);
+  }
+}
+
+function addBiggestChannelByMediaSize() {
+  let tBody = document.getElementById('biggestChannelTableMediaSize');
+
+  let apiEndpoint = '/api/stats/biggestchannels/?order=media_size';
+  const responseData = apiRequest(apiEndpoint, 'GET');
+
+  for (let i = 0; i < responseData.length; i++) {
+    const { id, name, media_size } = responseData[i];
+
+    let tableRow = buildChannelRow(id, name, humanFileSize(media_size));
+
+    tBody.appendChild(tableRow);
+  }
 }
 
 function clearLoading(dashBox) {
   dashBox.querySelector('#loading').remove();
 }
 
+function capitalizeFirstLetter(string) {
+  // source: https://stackoverflow.com/a/1026087
+  return string.charAt(0).toUpperCase() + string.slice(1);
+}
+
+function humanFileSize(size) {
+  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
+  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
+}
+
 function buildTile(titleText) {
   let tile = document.createElement('div');
   tile.classList.add('info-box-item');
@@ -89,226 +393,6 @@ function buildTileContenTable(content, rowsWanted) {
   return table;
 }
 
-function buildVideoTile(responseData) {
-  let tile = buildTile(`Video types: `);
-
-  const total = responseData.videos.total || 0;
-  const videos = responseData.videos.videos || 0;
-  const shorts = responseData.videos.shorts || 0;
-  const streams = responseData.videos.streams || 0;
-
-  const content = {
-    Videos: `${videos}/${total}`,
-    Shorts: `${shorts}/${total}`,
-    Streams: `${streams}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildChannelTile(responseData) {
-  let tile = buildTile(`Channels: `);
-
-  const total = responseData.channels.total || 0;
-  const subscribed = responseData.channels.sub_true || 0;
-
-  const content = {
-    Subscribed: `${subscribed}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildPlaylistTile(responseData) {
-  let tile = buildTile(`Playlists:`);
-
-  const total = responseData.playlists.total || 0;
-  const subscribed = responseData.playlists.sub_true || 0;
-
-  const content = {
-    Subscribed: `${subscribed}/${total}`,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function buildDownloadTile(responseData) {
-  let tile = buildTile('Downloads');
-
-  const pending = responseData.downloads.pending || 0;
-  const ignored = responseData.downloads.ignore || 0;
-
-  const content = {
-    Pending: pending,
-    Ignored: ignored,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function watchStats() {
-  let apiEndpoint = '/api/stats/watch/';
-  let responseData = apiRequest(apiEndpoint, 'GET');
-  let watchBox = document.getElementById('watchBox');
-  clearLoading(watchBox);
-
-  const { total, watched, unwatched } = responseData;
-
-  let firstCard = buildWatchTile('total', total);
-  watchBox.appendChild(firstCard);
-
-  let secondCard = buildWatchTile('watched', watched);
-  watchBox.appendChild(secondCard);
-
-  let thirdCard = buildWatchTile('unwatched', unwatched);
-  watchBox.appendChild(thirdCard);
-}
-
-function capitalizeFirstLetter(string) {
-  // source: https://stackoverflow.com/a/1026087
-  return string.charAt(0).toUpperCase() + string.slice(1);
-}
-
-function buildWatchTile(title, watchDetail) {
-  const items = watchDetail?.items ?? 0;
-  const duration = watchDetail?.duration ?? 0;
-  const duration_str = watchDetail?.duration_str ?? '0s';
-  const hasProgess = !!watchDetail?.progress;
-  const progress = (Number(watchDetail?.progress) * 100).toFixed(2) ?? '0';
-
-  let titleCapizalized = capitalizeFirstLetter(title);
-
-  if (hasProgess) {
-    titleCapizalized = `${progress}% ` + titleCapizalized;
-  }
-
-  let tile = buildTile(titleCapizalized);
-
-  const content = {
-    Videos: items,
-    Seconds: duration,
-    Playback: duration_str,
-  };
-
-  const table = buildTileContenTable(content, 3);
-
-  tile.appendChild(table);
-
-  return tile;
-}
-
-function downloadHist() {
-  let apiEndpoint = '/api/stats/downloadhist/';
-  let responseData = apiRequest(apiEndpoint, 'GET');
-  let histBox = document.getElementById('downHistBox');
-  clearLoading(histBox);
-  if (responseData.length === 0) {
-    let tile = buildTile('No recent downloads');
-    histBox.appendChild(tile);
-    return;
-  }
-
-  for (let i = 0; i < responseData.length; i++) {
-    const dailyStat = responseData[i];
-    let tile = buildDailyStat(dailyStat);
-    histBox.appendChild(tile);
-  }
-}
-
-function buildDailyStat(dailyStat) {
-  let tile = buildTile(dailyStat.date);
-  let message = document.createElement('p');
-  const isExactlyOne = dailyStat.count === 1;
-
-  let text = 'Videos';
-  if (isExactlyOne) {
-    text = 'Video';
-  }
-
-  message.innerText = `+${dailyStat.count} ${text}
-  ${humanFileSize(dailyStat.media_size)}`;
-
-  tile.appendChild(message);
-  return tile;
-}
-
-function humanFileSize(size) {
-  let i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
-  return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
-}
-
-function buildChannelRow(id, name, value) {
-  let tableRow = document.createElement('tr');
-
-  tableRow.innerHTML = `
-    <td class="agg-channel-name"><a href="/channel/${id}/">${name}</a></td>
-    <td class="agg-channel-right-align">${value}</td>
-  `;
-
-  return tableRow;
-}
-
-function addBiggestChannelByDocCount() {
-  let tBody = document.getElementById('biggestChannelTableVideos');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=doc_count';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, doc_count } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, doc_count);
-
-    tBody.appendChild(tableRow);
-  }
-}
-
-function addBiggestChannelByDuration() {
-  const tBody = document.getElementById('biggestChannelTableDuration');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=duration';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, duration_str } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, duration_str);
-
-    tBody.appendChild(tableRow);
-  }
-}
-
-function addBiggestChannelByMediaSize() {
-  let tBody = document.getElementById('biggestChannelTableMediaSize');
-
-  let apiEndpoint = '/api/stats/biggestchannels/?order=media_size';
-  const responseData = apiRequest(apiEndpoint, 'GET');
-
-  for (let i = 0; i < responseData.length; i++) {
-    const { id, name, media_size } = responseData[i];
-
-    let tableRow = buildChannelRow(id, name, humanFileSize(media_size));
-
-    tBody.appendChild(tableRow);
-  }
-}
-
 function biggestChannel() {
   addBiggestChannelByDocCount();
   addBiggestChannelByDuration();
@@ -317,6 +401,7 @@ function biggestChannel() {
 
 async function buildStats() {
   primaryStats();
+  secondaryStats();
   watchStats();
   downloadHist();
   biggestChannel();