From 2f05886c7bb71e56aed2a27e1231d5b3663d89f9 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Thu, 14 Apr 2022 21:58:31 +0100 Subject: [PATCH 1/8] FIX: Exit code incorrectly placed --- deploy.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) mode change 100644 => 100755 deploy.sh diff --git a/deploy.sh b/deploy.sh old mode 100644 new mode 100755 index 5eb6cd4..b505716 --- a/deploy.sh +++ b/deploy.sh @@ -1,3 +1,5 @@ +#!/bin/bash + function validate { if [[ $1 ]]; then @@ -21,10 +23,10 @@ function validate { } -exit 0 - if [[ $1 == "validate" ]]; then validate else echo "valid options are: validate" -fi \ No newline at end of file +fi + +exit 0 \ No newline at end of file From 8d0f2cc2a9133365180c587bc7f107efcbb075a9 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Thu, 14 Apr 2022 22:26:58 +0100 Subject: [PATCH 2/8] Creating requirements.txt to pin package versions --- tubearchivist-metrics/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tubearchivist-metrics/requirements.txt diff --git a/tubearchivist-metrics/requirements.txt b/tubearchivist-metrics/requirements.txt new file mode 100644 index 0000000..5902098 --- /dev/null +++ b/tubearchivist-metrics/requirements.txt @@ -0,0 +1,2 @@ +elasticsearch==8.1.2 +prometheus_client==0.14.1 From 76a8195b5923cfa1a59594c95e694e1a9e054ed9 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Thu, 14 Apr 2022 22:27:28 +0100 Subject: [PATCH 3/8] MVP created --- tubearchivist-metrics/environment.py | 32 ++++++++++++++ tubearchivist-metrics/esconnect.py | 30 +++++++++++++ tubearchivist-metrics/getmetrics.py | 11 +++++ tubearchivist-metrics/main.py | 63 ++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 tubearchivist-metrics/environment.py create mode 100644 tubearchivist-metrics/esconnect.py create mode 100644 tubearchivist-metrics/getmetrics.py create mode 100644 tubearchivist-metrics/main.py diff --git a/tubearchivist-metrics/environment.py b/tubearchivist-metrics/environment.py new file mode 100644 index 0000000..983effa --- /dev/null +++ b/tubearchivist-metrics/environment.py @@ -0,0 +1,32 @@ +""" +Functionality for setting up the environment for the metrics package. +Reads in environment variables for the application to use. +""" +import os + + +class AppConfig: + def __init__(self) -> None: + self.config = self.get_config() + + @staticmethod + def get_config(): + """ + Reads in environment variables for the application to use. + """ + + es_pass = os.environ.get("ES_PASSWORD") + es_user = os.environ.get("ES_USER", default="elastic") + es_url = os.environ.get("ES_URL", default="http://archivist-es:9200") + listen_port = os.environ.get("LISTEN_PORT", default="9934") + poll_interval = os.environ.get("POLL_INTERVAL", default="60") + + application = { + "es_url": es_url, + "es_user": es_user, + "es_pass": es_pass, + "listen_port": listen_port, + "poll_interval": poll_interval, + } + + return application diff --git a/tubearchivist-metrics/esconnect.py b/tubearchivist-metrics/esconnect.py new file mode 100644 index 0000000..094dc35 --- /dev/null +++ b/tubearchivist-metrics/esconnect.py @@ -0,0 +1,30 @@ +from elasticsearch import Elasticsearch +from environment import AppConfig + +config = AppConfig().config + + +class ElasticWrapper: + """ + makes calls to elastic search + returns response count + """ + + es_url = config["es_url"] + es_user = config["es_user"] + es_pass = config["es_pass"] + + es = Elasticsearch( + [es_url], + basic_auth=(es_user, es_pass), + timeout=30, + max_retries=10, + retry_on_timeout=True, + ) + + def get_count(index_name): + """ + Returns the number of documents in the index + """ + response = ElasticWrapper.es.count(index=index_name) + return response["count"] diff --git a/tubearchivist-metrics/getmetrics.py b/tubearchivist-metrics/getmetrics.py new file mode 100644 index 0000000..21497b0 --- /dev/null +++ b/tubearchivist-metrics/getmetrics.py @@ -0,0 +1,11 @@ +from esconnect import ElasticWrapper + + +class GetMetrics: + @staticmethod + def count(index_name): + + """Get count of documents from ES""" + result = ElasticWrapper.get_count(index_name) + print("Metric for " + index_name + ": " + str(result)) + return int(result) diff --git a/tubearchivist-metrics/main.py b/tubearchivist-metrics/main.py new file mode 100644 index 0000000..0d390ba --- /dev/null +++ b/tubearchivist-metrics/main.py @@ -0,0 +1,63 @@ +import time +from prometheus_client import start_http_server, Gauge, Enum + +from environment import AppConfig +from getmetrics import GetMetrics + +config = AppConfig().config + +# Print configuration on console when starting the application + +print("Configuration is currently set to:") +print("Elasticsearch URL: " + config["es_url"]) +print("Listen Port: " + config["listen_port"]) +print("Poll Interval (seconds): " + config["poll_interval"]) + + +class AppMetrics: + # fmt: off + def __init__(self, poll_interval=int(config["poll_interval"])): + + self.poll_interval = poll_interval + + # Metrics to expose + self.channel_count = Gauge("channel_count", "Number of channels") + self.playlist_count = Gauge("playlist_count", "Number of playlists") + self.download_count = Gauge("download_count", "Number of downloads") + self.download_queue = Gauge("download_queue", "Number of pending downloads") + self.subtitle_count = Gauge("subtitle_count", "Number of subtitles downloaded for videos") + # fmt: on + + def run_metrics_loop(self): + """ + Runs a loop that will update the metrics every second. + """ + while True: + self.retrieve_metrics() + time.sleep(self.poll_interval) + + def retrieve_metrics(self): + """ + Retrieves the metrics from the database and updates the metrics. + """ + self.channel_count.set(GetMetrics.count(index_name="ta_channel")) + self.playlist_count.set(GetMetrics.count(index_name="ta_playlist")) + self.download_queue.set(GetMetrics.count(index_name="ta_download")) + self.download_count.set(GetMetrics.count(index_name="ta_video")) + self.subtitle_count.set(GetMetrics.count(index_name="ta_subtitle")) + + +def main(): + """Main Entry Point""" + listen_port = int(config["listen_port"]) + poll_interval = int(config["poll_interval"]) + + app_metrics = AppMetrics( + poll_interval=poll_interval, + ) + start_http_server(listen_port) + app_metrics.run_metrics_loop() + + +if __name__ == "__main__": + main() From 6de6a458367abc218e07c0835d85a4a2be18993c Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Thu, 14 Apr 2022 22:28:03 +0100 Subject: [PATCH 4/8] README.md updated with guides on how to run, metrics that are exposed and how the gathering ocurrs. --- README.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5717484..8bab919 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,74 @@ Provide Tube Archivist metrics in Prometheus/OpenMetrics format This is an optional service as part of the Tube Archivist stack. -This is WIP +**This is WIP** +--- +  + +## Metrics reported +``` +channel_count = Number of channels +playlist_count = Number of playlists +download_count = Number of downloads +download_queue = Number of pending downloads +subtitle_count = Number of subtitles downloaded for videos + +``` ## Configuration +--- +### Environment variables +``` +ES_URL: The URL to your ElasticSearch server. Defaults to http://archivist-es:9200 +ES_USER: The username for authentication to ElasticSearch. Defaults to elastic +ES_PASSWORD: The password for authentication to ElasticSearch. No default is set. +LISTEN_PORT: The listen port for the metrics server to run on. Defaults to 9934 +POLL_INTERVAL: The interval in seconds for the data to be scraped from ElasticSearch. Defaults to 60 +``` +--- +### Running via Docker Compose (supported) +This metrics service is designed to be ran inside of docker. + +To view the main compose file for TubeArchivist, please see the main repository here: [TA Repo](https://github.com/bbilly1/tubearchivist) + +To add the metrics service in, place this into your compose file and update the environment variables as required. + +``` + archivist-metrics: + image: tubearchivist-metrics:latest + container_name: archivist-metrics + restart: always + environment: + - "ES_USER=elastic" + - "ES_PASSWORD=verysecret" + - "ES_URL=http://archivist-es:9200" + - "LISTEN_PORT=9934" + - "POLL_INTERVAL=60" + ports: + - 9934:9934 +``` +--- + +### Running via Standalone +Should you want to, you can install and run this without docker. + +To do so, clone this repo, install the python libraries with `pip3 install -r requirements.txt` + +then run `python3 tubearchivist-metrics/main.py` + +Environment variables can be passed in via normal syntax for your OS. + +--- + +## How are metrics gathered? + +Typically, a prometheus server will poll the HTTP endpoint of the metrics service to obtain its metrics. + +In most scenarios, a service will then retrieve the data for the metric, and then respond to the prometheus http call. However this can be quite harsh on databases and applications, especially when prometheus is polling every 15 seconds. + +To prevent performance issues and unncessecary load on ElasticSearch. We prefetch the metric information from ES every 60 seconds (default). The metric is then updated on the HTTP endpoint after we have retrieved the data and cached for prometheus to scrape. + +This means prometheus can scrape the endpoint every second if it likes, but no database calls to ES will be made until the polling interval is reached. + +If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable \ No newline at end of file From ace246386f10d05bc87f260f184d813532ee1f68 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Fri, 15 Apr 2022 01:12:21 +0100 Subject: [PATCH 5/8] Addition of working dockerfile --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..527e099 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.10-slim-buster +WORKDIR /app +COPY tubearchivist-metrics/requirements.txt . +RUN pip3 install -r requirements.txt +COPY . . +CMD ["python3", "-u", "tubearchivist-metrics/main.py"] \ No newline at end of file From 2b4be61d211f2eb25054f9582a18fae833edc86e Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Fri, 15 Apr 2022 01:12:37 +0100 Subject: [PATCH 6/8] Addition of example prometheus configuration --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 8bab919..f8b5e61 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,15 @@ Environment variables can be passed in via normal syntax for your OS. --- +### Prometheus example config +``` + - job_name: 'tubearchivist-metrics' + metrics_path: / + static_configs: + - targets: + - :9934 +``` +--- ## How are metrics gathered? Typically, a prometheus server will poll the HTTP endpoint of the metrics service to obtain its metrics. From 848c390bd79220a8388bafda54ffe198c286b1a6 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Fri, 15 Apr 2022 01:13:25 +0100 Subject: [PATCH 7/8] Remove unused import re-name metrics to be prefixed with yta_ added print log to show the loop is running as expected --- tubearchivist-metrics/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tubearchivist-metrics/main.py b/tubearchivist-metrics/main.py index 0d390ba..59cc207 100644 --- a/tubearchivist-metrics/main.py +++ b/tubearchivist-metrics/main.py @@ -1,5 +1,5 @@ import time -from prometheus_client import start_http_server, Gauge, Enum +from prometheus_client import start_http_server, Gauge from environment import AppConfig from getmetrics import GetMetrics @@ -21,11 +21,11 @@ class AppMetrics: self.poll_interval = poll_interval # Metrics to expose - self.channel_count = Gauge("channel_count", "Number of channels") - self.playlist_count = Gauge("playlist_count", "Number of playlists") - self.download_count = Gauge("download_count", "Number of downloads") - self.download_queue = Gauge("download_queue", "Number of pending downloads") - self.subtitle_count = Gauge("subtitle_count", "Number of subtitles downloaded for videos") + self.channel_count = Gauge("yta_channel_count", "Number of channels") + self.playlist_count = Gauge("yta_playlist_count", "Number of playlists") + self.download_count = Gauge("yta_download_count", "Number of downloads") + self.download_queue = Gauge("yta_download_queue", "Number of pending downloads") + self.subtitle_count = Gauge("yta_subtitle_count", "Number of subtitles downloaded for videos") # fmt: on def run_metrics_loop(self): @@ -40,6 +40,7 @@ class AppMetrics: """ Retrieves the metrics from the database and updates the metrics. """ + print("Obtaining Metrics from Elasticsearch") self.channel_count.set(GetMetrics.count(index_name="ta_channel")) self.playlist_count.set(GetMetrics.count(index_name="ta_playlist")) self.download_queue.set(GetMetrics.count(index_name="ta_download")) From d05f9aec5141242301a4425a6974d5a1f00097a2 Mon Sep 17 00:00:00 2001 From: Ainsey11 Date: Fri, 15 Apr 2022 01:13:55 +0100 Subject: [PATCH 8/8] Refactor of connection wrapper to handle errors connecting to ES properly. --- tubearchivist-metrics/esconnect.py | 71 +++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/tubearchivist-metrics/esconnect.py b/tubearchivist-metrics/esconnect.py index 094dc35..07abfe5 100644 --- a/tubearchivist-metrics/esconnect.py +++ b/tubearchivist-metrics/esconnect.py @@ -1,30 +1,59 @@ -from elasticsearch import Elasticsearch -from environment import AppConfig +from multiprocessing import AuthenticationError +from elasticsearch import ( + Elasticsearch, + ConnectionError, + ConnectionTimeout, + AuthenticationException, + AuthorizationException, +) -config = AppConfig().config +from environment import AppConfig +from time import sleep class ElasticWrapper: - """ - makes calls to elastic search - returns response count - """ - - es_url = config["es_url"] - es_user = config["es_user"] - es_pass = config["es_pass"] - - es = Elasticsearch( - [es_url], - basic_auth=(es_user, es_pass), - timeout=30, - max_retries=10, - retry_on_timeout=True, - ) + def handle_err(error): + print("Connection Error: " + str(error)) + print("There was a problem connecting to Elasticsearch") + print( + "Please see the error above. This may be as Elasticsearch is still starting up or a misconfiguration" + ) + print("Sleeping for 10 seconds...") + sleep(10) def get_count(index_name): """ Returns the number of documents in the index """ - response = ElasticWrapper.es.count(index=index_name) - return response["count"] + config = AppConfig().config + es_url = config["es_url"] + es_user = config["es_user"] + es_pass = config["es_pass"] + + es = Elasticsearch( + [es_url], + basic_auth=(es_user, es_pass), + timeout=10, + max_retries=12, + retry_on_timeout=True, + ) + + response = 0 + try: + response = es.count(index=index_name)["count"] + except AuthenticationException as e: + ElasticWrapper.handle_err(e) + except ConnectionError as e: + ElasticWrapper.handle_err(e) + except ConnectionTimeout as e: + ElasticWrapper.handle_err(e) + except AuthenticationError as e: + ElasticWrapper.handle_err(e) + except AuthorizationException as e: + ElasticWrapper.handle_err(e) + except: + print("Unknown error occurred. Check your credentials, url and try again.") + print("Sleeping for 10 seconds...") + sleep(10) + else: + return response