From 0bfbeb978d3d6be5cf983fdb88f275308db34dda Mon Sep 17 00:00:00 2001 From: tnewbury Date: Tue, 1 Jul 2025 13:55:31 +0100 Subject: [PATCH] Various tweaks and updates (#8) * Changed to use API rather than DB. Needs a lot of post change cleanup though * Updated readme * Grafana Dashboard JSON markup for grafana dash * Lots of tidying. Removal of old comments, added new comments. Updated base version of the docker container. fixed some spelling * Removed comments --------- Co-authored-by: tnewbury Co-authored-by: tnewbury --- Dockerfile | 4 +- README.md | 40 +- TubeArchivist.json | 689 +++++++++++++++++++++++++ tubearchivist-metrics/environment.py | 14 +- tubearchivist-metrics/esconnect.py | 60 --- tubearchivist-metrics/getmetrics.py | 12 +- tubearchivist-metrics/main.py | 71 ++- tubearchivist-metrics/requirements.txt | 7 +- tubearchivist-metrics/tascraper.py | 55 ++ 9 files changed, 849 insertions(+), 103 deletions(-) create mode 100644 TubeArchivist.json delete mode 100644 tubearchivist-metrics/esconnect.py create mode 100644 tubearchivist-metrics/tascraper.py diff --git a/Dockerfile b/Dockerfile index 2c2e52c..23e4b40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:3.10.4-slim-bullseye +FROM python:3.11.8-slim-bookworm WORKDIR /app COPY tubearchivist-metrics/requirements.txt . RUN pip3 install -r requirements.txt COPY . . -CMD ["python3", "-u", "tubearchivist-metrics/main.py"] \ No newline at end of file +CMD ["python3", "-u", "tubearchivist-metrics/main.py"] diff --git a/README.md b/README.md index b4a872d..3a4f4dd 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,27 @@ This is an optional service as part of the Tube Archivist stack. ## Metrics reported ``` -channel_count = Number of channels -playlist_count = Number of playlists -download_count = Number of downloads -download_queue = Number of pending downloads -subtitle_count = Number of subtitles downloaded for videos +yta_ignore_downloads, Total number of ignored videos +yta_pending_downloads, Total number of pending downloads +yta_pending_videos, Total number of pending video downloads +yta_pending_shorts, Total number of pending shorts downloads +yta_pending_streams, Total number of pending stream downloads + +yta_videos_total, Total number of videos + +yta_channel_total, Total number of channels +yta_channel_active, Total number of active channels +yta_channel_inactive, Total number of inactive channels +yta_channel_subscribed, Total number of subscribed channels +yta_channel_unsubscribed, Total number of unsuubscribed channels + + + +yta_playlists_total, Total number of playlists +yta_playlists_active, Total number of active playlists +yta_playlists_inactive, "Total number of inactive playlists +yta_playlists_subscribed, Total number of subscribed playlists +yta_playlists_unsubscribed, Total number of unsubscribed playlists ``` @@ -22,9 +38,8 @@ subtitle_count = Number of subtitles downloaded for videos --- ### Environment variables ``` -ES_URL: The URL to your ElasticSearch server. Defaults to http://archivist-es:9200 -ES_USER: The username for authentication to ElasticSearch. Defaults to elastic -ES_PASSWORD: The password for authentication to ElasticSearch. No default is set. +TA_URL: The URL to your TubeArchivist Server +TA_KEY: Your TubeArchivist API key LISTEN_PORT: The listen port for the metrics server to run on. Defaults to 9934 POLL_INTERVAL: The interval in seconds for the data to be scraped from ElasticSearch. Defaults to 60 ``` @@ -42,9 +57,8 @@ To add the metrics service in, place this into your compose file and update the container_name: archivist-metrics restart: always environment: - - "ES_USER=elastic" - - "ES_PASSWORD=verysecret" - - "ES_URL=http://archivist-es:9200" + - "TA_URL=http://tubearchivist.local" + - "TA_KEY="your ta api key" - "LISTEN_PORT=9934" - "POLL_INTERVAL=60" ports: @@ -78,8 +92,8 @@ Typically, a prometheus server will poll the HTTP endpoint of the metrics servic In most scenarios, a service will then retrieve the data for the metric, and then respond to the prometheus http call. However this can be quite harsh on databases and applications, especially when prometheus is polling every 15 seconds. -To prevent performance issues and unncessecary load on ElasticSearch. We prefetch the metric information from ES every 60 seconds (default). The metric is then updated on the HTTP endpoint after we have retrieved the data and cached for prometheus to scrape. +To prevent performance issues and unncessecary load on ElasticSearch. We prefetch the metric information from the API every 60 seconds (default). The metric is then updated on the HTTP endpoint after we have retrieved the data and cached for prometheus to scrape. -This means prometheus can scrape the endpoint every second if it likes, but no database calls to ES will be made until the polling interval is reached. +This means prometheus can scrape the endpoint every second if it likes, but no database calls to the API will be made until the polling interval is reached. If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable \ No newline at end of file diff --git a/TubeArchivist.json b/TubeArchivist.json new file mode 100644 index 0000000..ab7db28 --- /dev/null +++ b/TubeArchivist.json @@ -0,0 +1,689 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Subscription Counts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_playlists_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Playlist Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_playlists_active", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Active", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_playlists_subscribed", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Subscribed", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Playlist Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_channel_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_channel_active", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Active", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_channel_subscribed", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Subscribed", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_channel_unsubscribed", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Unsubscribed", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Channel Count", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "Download metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_pending_videos", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Videos", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_pending_shorts", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Shorts", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_pending_streams", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Streams", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_pending_downloads", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "All", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_videos_total", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total Videos", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Total videos", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "yta_ignore_downloads", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Ignored", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Ignored(all time downloads)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "TubeArchivist", + "uid": "eae0aa88-11cf-4198-9e4c-d2d010a05a88", + "version": 24 +} \ No newline at end of file diff --git a/tubearchivist-metrics/environment.py b/tubearchivist-metrics/environment.py index 983effa..c6154f3 100644 --- a/tubearchivist-metrics/environment.py +++ b/tubearchivist-metrics/environment.py @@ -15,16 +15,16 @@ class AppConfig: Reads in environment variables for the application to use. """ - es_pass = os.environ.get("ES_PASSWORD") - es_user = os.environ.get("ES_USER", default="elastic") - es_url = os.environ.get("ES_URL", default="http://archivist-es:9200") + + ta_key = os.environ.get("TA_KEY") + ta_url = os.environ.get("TA_URL") listen_port = os.environ.get("LISTEN_PORT", default="9934") - poll_interval = os.environ.get("POLL_INTERVAL", default="60") + poll_interval = os.environ.get("POLL_INTERVAL", default="120") application = { - "es_url": es_url, - "es_user": es_user, - "es_pass": es_pass, + "ta_key": ta_key, + "ta_url": ta_url, + "listen_port": listen_port, "poll_interval": poll_interval, } diff --git a/tubearchivist-metrics/esconnect.py b/tubearchivist-metrics/esconnect.py deleted file mode 100644 index 38ead53..0000000 --- a/tubearchivist-metrics/esconnect.py +++ /dev/null @@ -1,60 +0,0 @@ -from multiprocessing import AuthenticationError -from elasticsearch import ( - Elasticsearch, - ConnectionError, - ConnectionTimeout, - AuthenticationException, - AuthorizationException, -) - -from environment import AppConfig -from time import sleep - - -class ElasticWrapper: - def handle_err(error): - print("Connection Error: " + str(error)) - print("There was a problem connecting to Elasticsearch") - print( - "Please see the error above. This may be as Elasticsearch is still starting up or a misconfiguration" - ) - print("Sleeping for 10 seconds...") - sleep(10) - - def get_count(index_name): - """ - Returns the number of documents in the index - """ - config = AppConfig().config - es_url = config["es_url"] - es_user = config["es_user"] - es_pass = config["es_pass"] - - es = Elasticsearch( - [es_url], - basic_auth=(es_user, es_pass), - timeout=10, - max_retries=12, - retry_on_timeout=True, - ) - - response = 0 - try: - response = es.count(index=index_name)["count"] - except AuthenticationException as e: - ElasticWrapper.handle_err(e) - except ConnectionError as e: - ElasticWrapper.handle_err(e) - except ConnectionTimeout as e: - ElasticWrapper.handle_err(e) - except AuthenticationError as e: - ElasticWrapper.handle_err(e) - except AuthorizationException as e: - ElasticWrapper.handle_err(e) - except: - print("Unknown error occurred. Check your credentials, url and try again.") - print("Sleeping for 10 seconds...") - sleep(10) - else: - es.close() - return response diff --git a/tubearchivist-metrics/getmetrics.py b/tubearchivist-metrics/getmetrics.py index a02282a..4d836d2 100644 --- a/tubearchivist-metrics/getmetrics.py +++ b/tubearchivist-metrics/getmetrics.py @@ -1,11 +1,11 @@ -from esconnect import ElasticWrapper +from tascraper import APIWrapper class GetMetrics: @staticmethod - def count(index_name): + def count(index_name, keyvalue): - """Get count of documents from ES""" - result = ElasticWrapper.get_count(index_name) - print(f"Metric for {index_name}: {result}") - return int(result) + """Get count of documents from API""" + result = APIWrapper.get_count(index_name, keyvalue) + #print(f"Metric for {index_name}: {keyvalue}: {result}") + return result diff --git a/tubearchivist-metrics/main.py b/tubearchivist-metrics/main.py index d729b4b..74b140c 100644 --- a/tubearchivist-metrics/main.py +++ b/tubearchivist-metrics/main.py @@ -9,7 +9,7 @@ config = AppConfig().config # Print configuration on console when starting the application print("Configuration is currently set to:") -print(f'Elasticsearch URL: {config["es_url"]}') +print(f'TA URL: {config["ta_url"]}') print(f'Listen Port: {config["listen_port"]}') print(f'Polling interval (seconds): {config["poll_interval"]}') @@ -21,16 +21,38 @@ class AppMetrics: self.poll_interval = poll_interval # Metrics to expose - self.channel_count = Gauge("yta_channel_count", "Number of channels") - self.playlist_count = Gauge("yta_playlist_count", "Number of playlists") - self.download_count = Gauge("yta_download_count", "Number of downloads") - self.download_queue = Gauge("yta_download_queue", "Number of pending downloads") - self.subtitle_count = Gauge("yta_subtitle_count", "Number of subtitles downloaded for videos") + + self.ignore_downloads = Gauge("yta_ignore_downloads", "Total number of ignored videos") + self.pending_downloads = Gauge("yta_pending_downloads", "Total number of pending downloads") + self.pending_videos = Gauge("yta_pending_videos", "Total number of pending video downloads") + self.pending_shorts = Gauge("yta_pending_shorts", "Total number of pending shorts downloads") + self.pending_streams = Gauge("yta_pending_streams", "Total number of pending stream downloads") + #These 3 are sub, sub bits of the tree. I've done all this in a really hacky way that only supports a depth of 1, + #ideally needs a full rewrite + #self.watch_total = Gauge("yta_watch_total", "Total number of Videos") + #self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos") + #self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos") + self.videos_total = Gauge("yta_videos_total", "Total number of videos") + + + self.channel_total = Gauge("yta_channel_total", "Total number of channels") + self.channel_active = Gauge("yta_channel_active", "Total number of active channels") + self.channel_inactive = Gauge("yta_channel_inactive", "Total number of inactive channels") + self.channel_subscribed = Gauge("yta_channel_subscribed", "Total number of subscribed channels") + self.channel_unsubscribed =Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels") + + + + self.playlists_total = Gauge("yta_playlists_total", "Total number of playlists") + self.playlists_active = Gauge("yta_playlists_active", "Total number of active playlists") + self.playlists_inactive = Gauge("yta_playlists_inactive", "Total number of inactive playlists") + self.playlists_subscribed = Gauge("yta_playlists_subscribed", "Total number of subscribed playlists") + self.playlists_unsubscribed =Gauge("yta_playlists_unsubscribed", "Total number of unsubscribed playlists") # fmt: on def run_metrics_loop(self): """ - Runs a loop that will update the metrics every second. + Runs a loop that will update the metrics every poll_interval. """ while True: self.retrieve_metrics() @@ -40,12 +62,35 @@ class AppMetrics: """ Retrieves the metrics from the database and updates the metrics. """ - print("Obtaining Metrics from Elasticsearch") - self.channel_count.set(GetMetrics.count(index_name="ta_channel")) - self.playlist_count.set(GetMetrics.count(index_name="ta_playlist")) - self.download_queue.set(GetMetrics.count(index_name="ta_download")) - self.download_count.set(GetMetrics.count(index_name="ta_video")) - self.subtitle_count.set(GetMetrics.count(index_name="ta_subtitle")) + + print("Obtaining Metrics from API") + + self.pending_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending")) + self.ignore_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="ignore")) + self.pending_videos.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending_videos")) + self.pending_shorts.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="shorts")) + self.pending_streams.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="streams")) + #Matching 3 for the above commented lines + #self.watch_total.set(GetMetrics.count(index_name="/api/stats/watch/")) + #self.watch_unwatched.set(GetMetrics.count(index_name="/api/stats/watch/")) + #self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/")) + self.videos_total.set(GetMetrics.count(index_name="/api/stats/video/", keyvalue="doc_count")) + + + self.channel_total.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="doc_count")) + self.channel_active.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_true")) + self.channel_inactive.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_false")) + self.channel_subscribed.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="subscribed_true")) + self.channel_unsubscribed.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="subscribed_false")) + + self.playlists_total.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="doc_count")) + self.playlists_active.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_true")) + self.playlists_inactive.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_false")) + self.playlists_subscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_true")) + self.playlists_unsubscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false")) + + + def main(): diff --git a/tubearchivist-metrics/requirements.txt b/tubearchivist-metrics/requirements.txt index 5902098..a80cbd7 100644 --- a/tubearchivist-metrics/requirements.txt +++ b/tubearchivist-metrics/requirements.txt @@ -1,2 +1,5 @@ -elasticsearch==8.1.2 -prometheus_client==0.14.1 + +prometheus_client==0.17.1 +requests==2.25.1 + + diff --git a/tubearchivist-metrics/tascraper.py b/tubearchivist-metrics/tascraper.py new file mode 100644 index 0000000..41065f4 --- /dev/null +++ b/tubearchivist-metrics/tascraper.py @@ -0,0 +1,55 @@ +import requests +import json + +from environment import AppConfig +from time import sleep + +#url = "/api/video//" +#headers = {"Authorization": "Token xxxxxxxxxx"} +#response = requests.get(url, headers=headers) + + + + +class APIWrapper: + def handle_err(error): + #None of the below is used. TODO. + print("Connection Error: " + str(error)) + print("There was a problem connecting to the TA API") + print( + "Please see the error above. This may be TA is still starting up or a misconfiguration" + ) + print("Sleeping for 60 seconds...") + sleep(60) + + def get_count(index_name, keyvalue): + + config = AppConfig().config + ta_key = config["ta_key"] + ta_url = config["ta_url"] + + headers = {"Authorization": "Token " + ta_key} + + response = 0 + + try: + #print(ta_url + index_name) + #print(keyvalue) + #print("-------------------------------------------------------") + + getjson = requests.get(ta_url + index_name, headers=headers) + + jsonreturn = json.loads(getjson.content) + + + response = jsonreturn[keyvalue] + if response is None: + response = 0 + + + except: + print("No values from " + ta_url + index_name + keyvalue) + #this has turned into a general failure statement due to bad error management + + return response +