From 1a32b0a51d00d5f373438b98b7f12875e3d831b4 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 21 Dec 2022 18:51:31 +0700 Subject: [PATCH] cleanup, move redis to separate file --- tubearchivist/web/src/api_docker.py | 2 +- tubearchivist/web/src/ta_redis.py | 88 +++++++++++++++++++++++++ tubearchivist/web/src/webhook_docker.py | 4 +- tubearchivist/web/src/webhook_github.py | 84 +---------------------- 4 files changed, 92 insertions(+), 86 deletions(-) create mode 100644 tubearchivist/web/src/ta_redis.py diff --git a/tubearchivist/web/src/api_docker.py b/tubearchivist/web/src/api_docker.py index ef5c415..e1a95ee 100644 --- a/tubearchivist/web/src/api_docker.py +++ b/tubearchivist/web/src/api_docker.py @@ -27,7 +27,7 @@ class DockerBackup: def _get_image_stats(self): """return dict for image""" - response = requests.get(self.URL).json() + response = requests.get(self.URL, timeout=20).json() now = datetime.now() last_updated = response["last_updated"] diff --git a/tubearchivist/web/src/ta_redis.py b/tubearchivist/web/src/ta_redis.py new file mode 100644 index 0000000..06ae12b --- /dev/null +++ b/tubearchivist/web/src/ta_redis.py @@ -0,0 +1,88 @@ +"""handle redis interactions""" + +import json +from datetime import datetime + +import redis + + +class RedisBase: + """connection base for redis""" + + REDIS_HOST = "redis" + REDIS_PORT = 6379 + NAME_SPACE = "ta:" + + def __init__(self): + self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) + + +class TaskHandler(RedisBase): + """handle buildx task queue""" + + def __init__(self, repo_conf, tag_name=False): + super().__init__() + self.key = self.NAME_SPACE + "task:buildx" + self.repo_conf = repo_conf + self.tag_name = tag_name + + def create_task(self, task_name): + """create task""" + self.create_queue() + self.set_task(task_name) + self.set_pub() + + def create_queue(self): + """set initial json object for queue""" + if self.conn.execute_command(f"EXISTS {self.key}"): + print(f"{self.key} already exists") + return + + message = { + "created": int(datetime.now().strftime("%s")), + "tasks": {} + } + self.conn.execute_command( + "JSON.SET", self.key, ".", json.dumps(message) + ) + + def set_task(self, task_name): + """publish new task to queue""" + + user = self.repo_conf.get("gh_user") + repo = self.repo_conf.get("gh_repo") + build_command = self.build_command(task_name) + task = { + "timestamp": int(datetime.now().strftime("%s")), + "clone": f"https://github.com/{user}/{repo}.git", + "name": self.repo_conf.get("gh_repo"), + "build": build_command, + } + if task_name == "sync_es": + task.update({"clone": False}) + + self.conn.json().set(self.key, f".tasks.{repo}", task) + + def build_command(self, task_name): + """return build command""" + if not self.tag_name: + return self.repo_conf.get(task_name) + + all_commands = self.repo_conf.get(task_name) + + if all(isinstance(i, list) for i in all_commands): + to_build_commands = [] + for command in all_commands: + to_build_commands.append(self._replace_version(command)) + else: + to_build_commands = self._replace_version(all_commands) + + return to_build_commands + + def _replace_version(self, command): + """replace version in str""" + return [i.replace("$VERSION", self.tag_name) for i in command] + + def set_pub(self): + """set message to pub""" + self.conn.publish(self.key, self.repo_conf.get("gh_repo")) diff --git a/tubearchivist/web/src/webhook_docker.py b/tubearchivist/web/src/webhook_docker.py index 080101c..17dfef4 100644 --- a/tubearchivist/web/src/webhook_docker.py +++ b/tubearchivist/web/src/webhook_docker.py @@ -74,7 +74,7 @@ class DockerHook(WebhookBase): user = self.repo_conf.get("gh_user") repo = self.repo_conf.get("gh_repo") url = f"https://api.github.com/repos/{user}/{repo}/commits/master" - response = requests.get(url).json() + response = requests.get(url, timeout=20).json() commit_url = response["html_url"] first_line_message = response["commit"]["message"].split("\n")[0] @@ -83,7 +83,7 @@ class DockerHook(WebhookBase): @staticmethod def _forward_message(message_data, url): """forward message to discrod""" - response = requests.post(url, json=message_data) + response = requests.post(url, json=message_data, timeout=20) if not response.ok: print(response.json()) return {"success": False} diff --git a/tubearchivist/web/src/webhook_github.py b/tubearchivist/web/src/webhook_github.py index bbfb808..85e6762 100644 --- a/tubearchivist/web/src/webhook_github.py +++ b/tubearchivist/web/src/webhook_github.py @@ -8,9 +8,9 @@ from hmac import HMAC, compare_digest from bs4 import BeautifulSoup import requests -import redis from src.db import DatabaseConnect +from src.ta_redis import TaskHandler from src.webhook_base import WebhookBase @@ -544,85 +544,3 @@ class EsVersionSync: """build task for builder""" task = TaskHandler(self.repo_conf, tag_name=self.expected) task.create_task("sync_es") - - -class RedisBase: - """connection base for redis""" - - REDIS_HOST = "redis" - REDIS_PORT = 6379 - NAME_SPACE = "ta:" - - def __init__(self): - self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT) - - -class TaskHandler(RedisBase): - """handle buildx task queue""" - - def __init__(self, repo_conf, tag_name=False): - super().__init__() - self.key = self.NAME_SPACE + "task:buildx" - self.repo_conf = repo_conf - self.tag_name = tag_name - - def create_task(self, task_name): - """create task""" - self.create_queue() - self.set_task(task_name) - self.set_pub() - - def create_queue(self): - """set initial json object for queue""" - if self.conn.execute_command(f"EXISTS {self.key}"): - print(f"{self.key} already exists") - return - - message = { - "created": int(datetime.now().strftime("%s")), - "tasks": {} - } - self.conn.execute_command( - "JSON.SET", self.key, ".", json.dumps(message) - ) - - def set_task(self, task_name): - """publish new task to queue""" - - user = self.repo_conf.get("gh_user") - repo = self.repo_conf.get("gh_repo") - build_command = self.build_command(task_name) - task = { - "timestamp": int(datetime.now().strftime("%s")), - "clone": f"https://github.com/{user}/{repo}.git", - "name": self.repo_conf.get("gh_repo"), - "build": build_command, - } - if task_name == "sync_es": - task.update({"clone": False}) - - self.conn.json().set(self.key, f".tasks.{repo}", task) - - def build_command(self, task_name): - """return build command""" - if not self.tag_name: - return self.repo_conf.get(task_name) - - all_commands = self.repo_conf.get(task_name) - - if all(isinstance(i, list) for i in all_commands): - to_build_commands = [] - for command in all_commands: - to_build_commands.append(self._replace_version(command)) - else: - to_build_commands = self._replace_version(all_commands) - - return to_build_commands - - def _replace_version(self, command): - """replace version in str""" - return [i.replace("$VERSION", self.tag_name) for i in command] - - def set_pub(self): - """set message to pub""" - self.conn.publish(self.key, self.repo_conf.get("gh_repo"))