linting and clean up

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--line-length=120"]
+  - repo: https://github.com/pycqa/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+        name: isort (python)
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--profile", "black", "-l 120"]
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.2
+    hooks:
+      - id: flake8
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--max-complexity=10", "--max-line-length=120"]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell

CONTRIBUTING.md

@@ -9,3 +9,17 @@ If you notice something is not working as expected, check to see if it has been
 If it has not yet been disclosed, go ahead and create an issue.  
 If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. 
 
+## Dev setup
+
+Setup your environment, e.g. with python venv:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements-dev.txt
+```
+
+Setup pre-commit for linting:
+```bash
+pre-commit install
+```

requirements-dev.txt

@@ -1,3 +1,4 @@
 -r tubearchivist-metrics/requirements.txt
 pre-commit==4.2.0
 requirementscheck==0.0.6
+types-requests==2.32.4.20250611

tubearchivist-metrics/environment.py

@@ -2,6 +2,7 @@
 Functionality for setting up the environment for the metrics package.
 Reads in environment variables for the application to use.
 """
+
 import os
 
 
@@ -15,7 +16,6 @@ class AppConfig:
         Reads in environment variables for the application to use.
         """
 
-
         ta_key = os.environ.get("TA_KEY")
         ta_url = os.environ.get("TA_URL")
         listen_port = os.environ.get("LISTEN_PORT", default="9934")
@@ -24,7 +24,6 @@ class AppConfig:
         application = {
             "ta_key": ta_key,
             "ta_url": ta_url,
- 
             "listen_port": listen_port,
             "poll_interval": poll_interval,
         }

tubearchivist-metrics/getmetrics.py

@@ -4,8 +4,7 @@ from tascraper import APIWrapper
 class GetMetrics:
     @staticmethod
     def count(index_name, keyvalue):
-
         """Get count of documents from API"""
-        result = APIWrapper.get_count(index_name, keyvalue)
+        result = APIWrapper().get_count(index_name, keyvalue)
         # print(f"Metric for {index_name}: {keyvalue}: {result}")
         return result

tubearchivist-metrics/main.py

@@ -1,8 +1,8 @@
 import time
-from prometheus_client import start_http_server, Gauge
 
 from environment import AppConfig
 from getmetrics import GetMetrics
+from prometheus_client import Gauge, start_http_server
 
 config = AppConfig().config
 
@@ -27,22 +27,20 @@ class AppMetrics:
         self.pending_videos = Gauge("yta_pending_videos", "Total number of pending video downloads")
         self.pending_shorts = Gauge("yta_pending_shorts", "Total number of pending shorts downloads")
         self.pending_streams = Gauge("yta_pending_streams", "Total number of pending stream downloads")
-        #These 3 are sub, sub bits of the tree. I've done all this in a really hacky way that only supports a depth of 1, 
+        # These 3 are sub, sub bits of the tree.
+        # I've done all this in a really hacky way that only supports a depth of 1,
         # ideally needs a full rewrite
         # self.watch_total = Gauge("yta_watch_total", "Total number of Videos")
         # self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos")
         # self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos")
         self.videos_total = Gauge("yta_videos_total", "Total number of videos")
 
-
         self.channel_total = Gauge("yta_channel_total", "Total number of channels")
         self.channel_active = Gauge("yta_channel_active", "Total number of active channels")
         self.channel_inactive = Gauge("yta_channel_inactive", "Total number of inactive channels")
         self.channel_subscribed = Gauge("yta_channel_subscribed", "Total number of subscribed channels")
         self.channel_unsubscribed = Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels")
 
-        
-        
         self.playlists_total = Gauge("yta_playlists_total", "Total number of playlists")
         self.playlists_active = Gauge("yta_playlists_active", "Total number of active playlists")
         self.playlists_inactive = Gauge("yta_playlists_inactive", "Total number of inactive playlists")
@@ -76,7 +74,6 @@ class AppMetrics:
         # self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/"))
         self.videos_total.set(GetMetrics.count(index_name="/api/stats/video/", keyvalue="doc_count"))
 
-
         self.channel_total.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="doc_count"))
         self.channel_active.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_true"))
         self.channel_inactive.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_false"))
@@ -87,10 +84,9 @@ class AppMetrics:
         self.playlists_active.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_true"))
         self.playlists_inactive.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_false"))
         self.playlists_subscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_true"))
-        self.playlists_unsubscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false"))
-        
-        
-
+        self.playlists_unsubscribed.set(
+            GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false")
+        )
 
 
 def main():

tubearchivist-metrics/requirements.txt

@@ -1,5 +1,2 @@
-
-
-
 prometheus_client==0.22.1
 requests==2.32.4

tubearchivist-metrics/tascraper.py

@@ -1,28 +1,25 @@
-import requests
 import json
-
-from environment import AppConfig
 from time import sleep
 
+import requests
+from environment import AppConfig
+
 # url = "/api/video/<video-id>/"
 # headers = {"Authorization": "Token xxxxxxxxxx"}
 # response = requests.get(url, headers=headers)
 
 
-
-
 class APIWrapper:
-    def handle_err(error):
+
+    def handle_err(self, error):
         # None of the below is used. TODO.
         print("Connection Error: " + str(error))
         print("There was a problem connecting to the TA API")
-        print(
-            "Please see the error above. This may be TA is still starting up or a misconfiguration"
-        )
+        print("Please see the error above. This may be TA is still starting up or a misconfiguration")
         print("Sleeping for 60 seconds...")
         sleep(60)
 
-    def get_count(index_name, keyvalue):
+    def get_count(self, index_name, keyvalue):
 
         config = AppConfig().config
         ta_key = config["ta_key"]
@@ -37,19 +34,16 @@ class APIWrapper:
             # print(keyvalue)
             # print("-------------------------------------------------------")
 
-            getjson = requests.get(ta_url + index_name, headers=headers)
+            getjson = requests.get(ta_url + index_name, headers=headers, timeout=30)
 
             jsonreturn = json.loads(getjson.content)
 
-        
             response = jsonreturn[keyvalue]
             if response is None:
                 response = 0
 
-            
-        except:
+        except Exception:
             print("No values from " + ta_url + index_name + keyvalue)
             # this has turned into a general failure statement due to bad error management
 
         return response
-