From b6732971b44b4a31d7d336946b1682b75638e009 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 1 Jul 2025 20:17:02 +0700 Subject: [PATCH] linting and clean up --- .gitignore | 2 +- .pre-commit-config.yaml | 31 +++++++++++++++++ CONTRIBUTING.md | 14 ++++++++ LICENSE | 2 +- README.md | 2 +- TubeArchivist.json | 2 +- requirements-dev.txt | 1 + tubearchivist-metrics/environment.py | 3 +- tubearchivist-metrics/getmetrics.py | 5 ++- tubearchivist-metrics/main.py | 44 +++++++++++------------- tubearchivist-metrics/requirements.txt | 3 -- tubearchivist-metrics/tascraper.py | 46 +++++++++++--------------- 12 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.gitignore b/.gitignore index ed8ebf5..bee8a64 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -__pycache__ \ No newline at end of file +__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fd19fb7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + alias: python + files: ^tubearchivist-metrics/ + args: ["--line-length=120"] + - repo: https://github.com/pycqa/isort + rev: 6.0.1 + hooks: + - id: isort + name: isort (python) + alias: python + files: ^tubearchivist-metrics/ + args: ["--profile", "black", "-l 120"] + - repo: https://github.com/pycqa/flake8 + rev: 7.1.2 + hooks: + - id: flake8 + alias: python + files: ^tubearchivist-metrics/ + args: ["--max-complexity=10", "--max-line-length=120"] + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4639c4..be6ca3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,3 +9,17 @@ If you notice something is not working as expected, check to see if it has been If it has not yet been disclosed, go ahead and create an issue. If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. +## Dev setup + +Setup your environment, e.g. with python venv: + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements-dev.txt +``` + +Setup pre-commit for linting: +```bash +pre-commit install +``` diff --git a/LICENSE b/LICENSE index 20d40b6..94a9ed0 100644 --- a/LICENSE +++ b/LICENSE @@ -671,4 +671,4 @@ into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read -. \ No newline at end of file +. diff --git a/README.md b/README.md index 3a4f4dd..c2e9421 100644 --- a/README.md +++ b/README.md @@ -96,4 +96,4 @@ To prevent performance issues and unncessecary load on ElasticSearch. We prefetc This means prometheus can scrape the endpoint every second if it likes, but no database calls to the API will be made until the polling interval is reached. -If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable \ No newline at end of file +If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable diff --git a/TubeArchivist.json b/TubeArchivist.json index ab7db28..7437279 100644 --- a/TubeArchivist.json +++ b/TubeArchivist.json @@ -686,4 +686,4 @@ "title": "TubeArchivist", "uid": "eae0aa88-11cf-4198-9e4c-d2d010a05a88", "version": 24 -} \ No newline at end of file +} diff --git a/requirements-dev.txt b/requirements-dev.txt index c72356e..53d0f1f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ -r tubearchivist-metrics/requirements.txt pre-commit==4.2.0 requirementscheck==0.0.6 +types-requests==2.32.4.20250611 diff --git a/tubearchivist-metrics/environment.py b/tubearchivist-metrics/environment.py index c6154f3..419d208 100644 --- a/tubearchivist-metrics/environment.py +++ b/tubearchivist-metrics/environment.py @@ -2,6 +2,7 @@ Functionality for setting up the environment for the metrics package. Reads in environment variables for the application to use. """ + import os @@ -15,7 +16,6 @@ class AppConfig: Reads in environment variables for the application to use. """ - ta_key = os.environ.get("TA_KEY") ta_url = os.environ.get("TA_URL") listen_port = os.environ.get("LISTEN_PORT", default="9934") @@ -24,7 +24,6 @@ class AppConfig: application = { "ta_key": ta_key, "ta_url": ta_url, - "listen_port": listen_port, "poll_interval": poll_interval, } diff --git a/tubearchivist-metrics/getmetrics.py b/tubearchivist-metrics/getmetrics.py index 4d836d2..fab71a0 100644 --- a/tubearchivist-metrics/getmetrics.py +++ b/tubearchivist-metrics/getmetrics.py @@ -4,8 +4,7 @@ from tascraper import APIWrapper class GetMetrics: @staticmethod def count(index_name, keyvalue): - """Get count of documents from API""" - result = APIWrapper.get_count(index_name, keyvalue) - #print(f"Metric for {index_name}: {keyvalue}: {result}") + result = APIWrapper().get_count(index_name, keyvalue) + # print(f"Metric for {index_name}: {keyvalue}: {result}") return result diff --git a/tubearchivist-metrics/main.py b/tubearchivist-metrics/main.py index 74b140c..5061288 100644 --- a/tubearchivist-metrics/main.py +++ b/tubearchivist-metrics/main.py @@ -1,8 +1,8 @@ import time -from prometheus_client import start_http_server, Gauge from environment import AppConfig from getmetrics import GetMetrics +from prometheus_client import Gauge, start_http_server config = AppConfig().config @@ -21,33 +21,31 @@ class AppMetrics: self.poll_interval = poll_interval # Metrics to expose - + self.ignore_downloads = Gauge("yta_ignore_downloads", "Total number of ignored videos") self.pending_downloads = Gauge("yta_pending_downloads", "Total number of pending downloads") self.pending_videos = Gauge("yta_pending_videos", "Total number of pending video downloads") self.pending_shorts = Gauge("yta_pending_shorts", "Total number of pending shorts downloads") self.pending_streams = Gauge("yta_pending_streams", "Total number of pending stream downloads") - #These 3 are sub, sub bits of the tree. I've done all this in a really hacky way that only supports a depth of 1, - #ideally needs a full rewrite - #self.watch_total = Gauge("yta_watch_total", "Total number of Videos") - #self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos") - #self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos") + # These 3 are sub, sub bits of the tree. + # I've done all this in a really hacky way that only supports a depth of 1, + # ideally needs a full rewrite + # self.watch_total = Gauge("yta_watch_total", "Total number of Videos") + # self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos") + # self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos") self.videos_total = Gauge("yta_videos_total", "Total number of videos") - self.channel_total = Gauge("yta_channel_total", "Total number of channels") self.channel_active = Gauge("yta_channel_active", "Total number of active channels") self.channel_inactive = Gauge("yta_channel_inactive", "Total number of inactive channels") self.channel_subscribed = Gauge("yta_channel_subscribed", "Total number of subscribed channels") - self.channel_unsubscribed =Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels") - - - + self.channel_unsubscribed = Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels") + self.playlists_total = Gauge("yta_playlists_total", "Total number of playlists") self.playlists_active = Gauge("yta_playlists_active", "Total number of active playlists") self.playlists_inactive = Gauge("yta_playlists_inactive", "Total number of inactive playlists") self.playlists_subscribed = Gauge("yta_playlists_subscribed", "Total number of subscribed playlists") - self.playlists_unsubscribed =Gauge("yta_playlists_unsubscribed", "Total number of unsubscribed playlists") + self.playlists_unsubscribed = Gauge("yta_playlists_unsubscribed", "Total number of unsubscribed playlists") # fmt: on def run_metrics_loop(self): @@ -62,20 +60,19 @@ class AppMetrics: """ Retrieves the metrics from the database and updates the metrics. """ - + print("Obtaining Metrics from API") - + self.pending_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending")) self.ignore_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="ignore")) self.pending_videos.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending_videos")) self.pending_shorts.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="shorts")) self.pending_streams.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="streams")) - #Matching 3 for the above commented lines - #self.watch_total.set(GetMetrics.count(index_name="/api/stats/watch/")) - #self.watch_unwatched.set(GetMetrics.count(index_name="/api/stats/watch/")) - #self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/")) + # Matching 3 for the above commented lines + # self.watch_total.set(GetMetrics.count(index_name="/api/stats/watch/")) + # self.watch_unwatched.set(GetMetrics.count(index_name="/api/stats/watch/")) + # self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/")) self.videos_total.set(GetMetrics.count(index_name="/api/stats/video/", keyvalue="doc_count")) - self.channel_total.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="doc_count")) self.channel_active.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_true")) @@ -87,10 +84,9 @@ class AppMetrics: self.playlists_active.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_true")) self.playlists_inactive.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_false")) self.playlists_subscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_true")) - self.playlists_unsubscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false")) - - - + self.playlists_unsubscribed.set( + GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false") + ) def main(): diff --git a/tubearchivist-metrics/requirements.txt b/tubearchivist-metrics/requirements.txt index a9410b2..51e53ea 100644 --- a/tubearchivist-metrics/requirements.txt +++ b/tubearchivist-metrics/requirements.txt @@ -1,5 +1,2 @@ - - - prometheus_client==0.22.1 requests==2.32.4 diff --git a/tubearchivist-metrics/tascraper.py b/tubearchivist-metrics/tascraper.py index 41065f4..58cf97d 100644 --- a/tubearchivist-metrics/tascraper.py +++ b/tubearchivist-metrics/tascraper.py @@ -1,55 +1,49 @@ -import requests import json - -from environment import AppConfig from time import sleep -#url = "/api/video//" -#headers = {"Authorization": "Token xxxxxxxxxx"} -#response = requests.get(url, headers=headers) - +import requests +from environment import AppConfig +# url = "/api/video//" +# headers = {"Authorization": "Token xxxxxxxxxx"} +# response = requests.get(url, headers=headers) class APIWrapper: - def handle_err(error): - #None of the below is used. TODO. + + def handle_err(self, error): + # None of the below is used. TODO. print("Connection Error: " + str(error)) print("There was a problem connecting to the TA API") - print( - "Please see the error above. This may be TA is still starting up or a misconfiguration" - ) + print("Please see the error above. This may be TA is still starting up or a misconfiguration") print("Sleeping for 60 seconds...") sleep(60) - def get_count(index_name, keyvalue): + def get_count(self, index_name, keyvalue): config = AppConfig().config ta_key = config["ta_key"] ta_url = config["ta_url"] headers = {"Authorization": "Token " + ta_key} - + response = 0 try: - #print(ta_url + index_name) - #print(keyvalue) - #print("-------------------------------------------------------") - - getjson = requests.get(ta_url + index_name, headers=headers) - + # print(ta_url + index_name) + # print(keyvalue) + # print("-------------------------------------------------------") + + getjson = requests.get(ta_url + index_name, headers=headers, timeout=30) + jsonreturn = json.loads(getjson.content) - - + response = jsonreturn[keyvalue] if response is None: response = 0 - - except: + except Exception: print("No values from " + ta_url + index_name + keyvalue) - #this has turned into a general failure statement due to bad error management + # this has turned into a general failure statement due to bad error management return response -