linting and clean up

.gitignore

@@ -1 +1 @@
-__pycache__
+__pycache__

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--line-length=120"]
+  - repo: https://github.com/pycqa/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort
+        name: isort (python)
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--profile", "black", "-l 120"]
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.2
+    hooks:
+      - id: flake8
+        alias: python
+        files: ^tubearchivist-metrics/
+        args: ["--max-complexity=10", "--max-line-length=120"]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell

CONTRIBUTING.md

@@ -9,3 +9,17 @@ If you notice something is not working as expected, check to see if it has been
 If it has not yet been disclosed, go ahead and create an issue.  
 If the issue doesn't move forward due to a lack of response, I assume it's solved and will close it after some time to keep the list fresh. 
 
+## Dev setup
+
+Setup your environment, e.g. with python venv:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements-dev.txt
+```
+
+Setup pre-commit for linting:
+```bash
+pre-commit install
+```

LICENSE

@@ -671,4 +671,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

README.md

@@ -96,4 +96,4 @@ To prevent performance issues and unncessecary load on ElasticSearch. We prefetc
 
 This means prometheus can scrape the endpoint every second if it likes, but no database calls to the API will be made until the polling interval is reached.
 
-If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable
+If you require more granular polling, you can update the `POLLING_INTERVAL` environment variable

TubeArchivist.json

@@ -686,4 +686,4 @@
   "title": "TubeArchivist",
   "uid": "eae0aa88-11cf-4198-9e4c-d2d010a05a88",
   "version": 24
-}
+}

requirements-dev.txt

@@ -1,3 +1,4 @@
 -r tubearchivist-metrics/requirements.txt
 pre-commit==4.2.0
 requirementscheck==0.0.6
+types-requests==2.32.4.20250611

tubearchivist-metrics/environment.py

@@ -2,6 +2,7 @@
 Functionality for setting up the environment for the metrics package.
 Reads in environment variables for the application to use.
 """
+
 import os
 
 
@@ -15,7 +16,6 @@ class AppConfig:
         Reads in environment variables for the application to use.
         """
 
-
         ta_key = os.environ.get("TA_KEY")
         ta_url = os.environ.get("TA_URL")
         listen_port = os.environ.get("LISTEN_PORT", default="9934")
@@ -24,7 +24,6 @@ class AppConfig:
         application = {
             "ta_key": ta_key,
             "ta_url": ta_url,
- 
             "listen_port": listen_port,
             "poll_interval": poll_interval,
         }

tubearchivist-metrics/getmetrics.py

@@ -4,8 +4,7 @@ from tascraper import APIWrapper
 class GetMetrics:
     @staticmethod
     def count(index_name, keyvalue):
-
         """Get count of documents from API"""
-        result = APIWrapper.get_count(index_name, keyvalue)
+        result = APIWrapper().get_count(index_name, keyvalue)
-        #print(f"Metric for {index_name}: {keyvalue}: {result}")
+        # print(f"Metric for {index_name}: {keyvalue}: {result}")
         return result

tubearchivist-metrics/main.py

@@ -1,8 +1,8 @@
 import time
-from prometheus_client import start_http_server, Gauge
 
 from environment import AppConfig
 from getmetrics import GetMetrics
+from prometheus_client import Gauge, start_http_server
 
 config = AppConfig().config
 
@@ -21,33 +21,31 @@ class AppMetrics:
         self.poll_interval = poll_interval
 
         # Metrics to expose
- 
+
         self.ignore_downloads = Gauge("yta_ignore_downloads", "Total number of ignored videos")
         self.pending_downloads = Gauge("yta_pending_downloads", "Total number of pending downloads")
         self.pending_videos = Gauge("yta_pending_videos", "Total number of pending video downloads")
         self.pending_shorts = Gauge("yta_pending_shorts", "Total number of pending shorts downloads")
         self.pending_streams = Gauge("yta_pending_streams", "Total number of pending stream downloads")
-        #These 3 are sub, sub bits of the tree. I've done all this in a really hacky way that only supports a depth of 1, 
+        # These 3 are sub, sub bits of the tree.
-        #ideally needs a full rewrite
+        # I've done all this in a really hacky way that only supports a depth of 1,
-        #self.watch_total = Gauge("yta_watch_total", "Total number of Videos")
+        # ideally needs a full rewrite
-        #self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos")
+        # self.watch_total = Gauge("yta_watch_total", "Total number of Videos")
-        #self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos")
+        # self.watch_unwatched = Gauge("yta_watch_unwatched", "Total number of unwatched videos")
+        # self.watch_watched = Gauge("yta_watch_watched", "Total number of watched videos")
         self.videos_total = Gauge("yta_videos_total", "Total number of videos")
-        
 
         self.channel_total = Gauge("yta_channel_total", "Total number of channels")
         self.channel_active = Gauge("yta_channel_active", "Total number of active channels")
         self.channel_inactive = Gauge("yta_channel_inactive", "Total number of inactive channels")
         self.channel_subscribed = Gauge("yta_channel_subscribed", "Total number of subscribed channels")
-        self.channel_unsubscribed =Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels")
+        self.channel_unsubscribed = Gauge("yta_channel_unsubscribed", "Total number of unsuubscribed channels")
-        
+
-        
-        
         self.playlists_total = Gauge("yta_playlists_total", "Total number of playlists")
         self.playlists_active = Gauge("yta_playlists_active", "Total number of active playlists")
         self.playlists_inactive = Gauge("yta_playlists_inactive", "Total number of inactive playlists")
         self.playlists_subscribed = Gauge("yta_playlists_subscribed", "Total number of subscribed playlists")
-        self.playlists_unsubscribed =Gauge("yta_playlists_unsubscribed", "Total number of unsubscribed playlists")
+        self.playlists_unsubscribed = Gauge("yta_playlists_unsubscribed", "Total number of unsubscribed playlists")
     # fmt: on
 
     def run_metrics_loop(self):
@@ -62,20 +60,19 @@ class AppMetrics:
         """
         Retrieves the metrics from the database and updates the metrics.
         """
-        
+
         print("Obtaining Metrics from API")
-        
+
         self.pending_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending"))
         self.ignore_downloads.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="ignore"))
         self.pending_videos.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="pending_videos"))
         self.pending_shorts.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="shorts"))
         self.pending_streams.set(GetMetrics.count(index_name="/api/stats/download/", keyvalue="streams"))
-        #Matching 3 for the above commented lines
+        # Matching 3 for the above commented lines
-        #self.watch_total.set(GetMetrics.count(index_name="/api/stats/watch/"))
+        # self.watch_total.set(GetMetrics.count(index_name="/api/stats/watch/"))
-        #self.watch_unwatched.set(GetMetrics.count(index_name="/api/stats/watch/"))
+        # self.watch_unwatched.set(GetMetrics.count(index_name="/api/stats/watch/"))
-        #self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/"))
+        # self.watch_watched.set(GetMetrics.count(index_name="/api/stats/watch/"))
         self.videos_total.set(GetMetrics.count(index_name="/api/stats/video/", keyvalue="doc_count"))
-        
 
         self.channel_total.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="doc_count"))
         self.channel_active.set(GetMetrics.count(index_name="/api/stats/channel/", keyvalue="active_true"))
@@ -87,10 +84,9 @@ class AppMetrics:
         self.playlists_active.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_true"))
         self.playlists_inactive.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="active_false"))
         self.playlists_subscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_true"))
-        self.playlists_unsubscribed.set(GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false"))
+        self.playlists_unsubscribed.set(
-        
+            GetMetrics.count(index_name="/api/stats/playlist/", keyvalue="subscribed_false")
-        
+        )
-
 
 
 def main():

tubearchivist-metrics/requirements.txt

@@ -1,5 +1,2 @@
-
-
-
 prometheus_client==0.22.1
 requests==2.32.4

tubearchivist-metrics/tascraper.py

@@ -1,55 +1,49 @@
-import requests
 import json
-
-from environment import AppConfig
 from time import sleep
 
-#url = "/api/video/<video-id>/"
+import requests
-#headers = {"Authorization": "Token xxxxxxxxxx"}
+from environment import AppConfig
-#response = requests.get(url, headers=headers)
-
 
+# url = "/api/video/<video-id>/"
+# headers = {"Authorization": "Token xxxxxxxxxx"}
+# response = requests.get(url, headers=headers)
 
 
 class APIWrapper:
-    def handle_err(error):
+
-        #None of the below is used. TODO.
+    def handle_err(self, error):
+        # None of the below is used. TODO.
         print("Connection Error: " + str(error))
         print("There was a problem connecting to the TA API")
-        print(
+        print("Please see the error above. This may be TA is still starting up or a misconfiguration")
-            "Please see the error above. This may be TA is still starting up or a misconfiguration"
-        )
         print("Sleeping for 60 seconds...")
         sleep(60)
 
-    def get_count(index_name, keyvalue):
+    def get_count(self, index_name, keyvalue):
 
         config = AppConfig().config
         ta_key = config["ta_key"]
         ta_url = config["ta_url"]
 
         headers = {"Authorization": "Token " + ta_key}
-       
+
         response = 0
 
         try:
-            #print(ta_url + index_name)
+            # print(ta_url + index_name)
-            #print(keyvalue)
+            # print(keyvalue)
-            #print("-------------------------------------------------------")
+            # print("-------------------------------------------------------")
-        
+
-            getjson = requests.get(ta_url + index_name, headers=headers)
+            getjson = requests.get(ta_url + index_name, headers=headers, timeout=30)
-         
+
             jsonreturn = json.loads(getjson.content)
-          
+
-        
             response = jsonreturn[keyvalue]
             if response is None:
                 response = 0
 
-            
+        except Exception:
-        except:
             print("No values from " + ta_url + index_name + keyvalue)
-            #this has turned into a general failure statement due to bad error management
+            # this has turned into a general failure statement due to bad error management
 
         return response
-