mirror of
https://github.com/tubearchivist/tubearchivist-server.git
synced 2024-11-14 16:10:12 +00:00
cleanup, move redis to separate file
This commit is contained in:
parent
a43ef63277
commit
1a32b0a51d
@ -27,7 +27,7 @@ class DockerBackup:
|
|||||||
|
|
||||||
def _get_image_stats(self):
|
def _get_image_stats(self):
|
||||||
"""return dict for image"""
|
"""return dict for image"""
|
||||||
response = requests.get(self.URL).json()
|
response = requests.get(self.URL, timeout=20).json()
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
|
|
||||||
last_updated = response["last_updated"]
|
last_updated = response["last_updated"]
|
||||||
|
88
tubearchivist/web/src/ta_redis.py
Normal file
88
tubearchivist/web/src/ta_redis.py
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
"""handle redis interactions"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
import redis
|
||||||
|
|
||||||
|
|
||||||
|
class RedisBase:
|
||||||
|
"""connection base for redis"""
|
||||||
|
|
||||||
|
REDIS_HOST = "redis"
|
||||||
|
REDIS_PORT = 6379
|
||||||
|
NAME_SPACE = "ta:"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskHandler(RedisBase):
|
||||||
|
"""handle buildx task queue"""
|
||||||
|
|
||||||
|
def __init__(self, repo_conf, tag_name=False):
|
||||||
|
super().__init__()
|
||||||
|
self.key = self.NAME_SPACE + "task:buildx"
|
||||||
|
self.repo_conf = repo_conf
|
||||||
|
self.tag_name = tag_name
|
||||||
|
|
||||||
|
def create_task(self, task_name):
|
||||||
|
"""create task"""
|
||||||
|
self.create_queue()
|
||||||
|
self.set_task(task_name)
|
||||||
|
self.set_pub()
|
||||||
|
|
||||||
|
def create_queue(self):
|
||||||
|
"""set initial json object for queue"""
|
||||||
|
if self.conn.execute_command(f"EXISTS {self.key}"):
|
||||||
|
print(f"{self.key} already exists")
|
||||||
|
return
|
||||||
|
|
||||||
|
message = {
|
||||||
|
"created": int(datetime.now().strftime("%s")),
|
||||||
|
"tasks": {}
|
||||||
|
}
|
||||||
|
self.conn.execute_command(
|
||||||
|
"JSON.SET", self.key, ".", json.dumps(message)
|
||||||
|
)
|
||||||
|
|
||||||
|
def set_task(self, task_name):
|
||||||
|
"""publish new task to queue"""
|
||||||
|
|
||||||
|
user = self.repo_conf.get("gh_user")
|
||||||
|
repo = self.repo_conf.get("gh_repo")
|
||||||
|
build_command = self.build_command(task_name)
|
||||||
|
task = {
|
||||||
|
"timestamp": int(datetime.now().strftime("%s")),
|
||||||
|
"clone": f"https://github.com/{user}/{repo}.git",
|
||||||
|
"name": self.repo_conf.get("gh_repo"),
|
||||||
|
"build": build_command,
|
||||||
|
}
|
||||||
|
if task_name == "sync_es":
|
||||||
|
task.update({"clone": False})
|
||||||
|
|
||||||
|
self.conn.json().set(self.key, f".tasks.{repo}", task)
|
||||||
|
|
||||||
|
def build_command(self, task_name):
|
||||||
|
"""return build command"""
|
||||||
|
if not self.tag_name:
|
||||||
|
return self.repo_conf.get(task_name)
|
||||||
|
|
||||||
|
all_commands = self.repo_conf.get(task_name)
|
||||||
|
|
||||||
|
if all(isinstance(i, list) for i in all_commands):
|
||||||
|
to_build_commands = []
|
||||||
|
for command in all_commands:
|
||||||
|
to_build_commands.append(self._replace_version(command))
|
||||||
|
else:
|
||||||
|
to_build_commands = self._replace_version(all_commands)
|
||||||
|
|
||||||
|
return to_build_commands
|
||||||
|
|
||||||
|
def _replace_version(self, command):
|
||||||
|
"""replace version in str"""
|
||||||
|
return [i.replace("$VERSION", self.tag_name) for i in command]
|
||||||
|
|
||||||
|
def set_pub(self):
|
||||||
|
"""set message to pub"""
|
||||||
|
self.conn.publish(self.key, self.repo_conf.get("gh_repo"))
|
@ -74,7 +74,7 @@ class DockerHook(WebhookBase):
|
|||||||
user = self.repo_conf.get("gh_user")
|
user = self.repo_conf.get("gh_user")
|
||||||
repo = self.repo_conf.get("gh_repo")
|
repo = self.repo_conf.get("gh_repo")
|
||||||
url = f"https://api.github.com/repos/{user}/{repo}/commits/master"
|
url = f"https://api.github.com/repos/{user}/{repo}/commits/master"
|
||||||
response = requests.get(url).json()
|
response = requests.get(url, timeout=20).json()
|
||||||
commit_url = response["html_url"]
|
commit_url = response["html_url"]
|
||||||
first_line_message = response["commit"]["message"].split("\n")[0]
|
first_line_message = response["commit"]["message"].split("\n")[0]
|
||||||
|
|
||||||
@ -83,7 +83,7 @@ class DockerHook(WebhookBase):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _forward_message(message_data, url):
|
def _forward_message(message_data, url):
|
||||||
"""forward message to discrod"""
|
"""forward message to discrod"""
|
||||||
response = requests.post(url, json=message_data)
|
response = requests.post(url, json=message_data, timeout=20)
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
print(response.json())
|
print(response.json())
|
||||||
return {"success": False}
|
return {"success": False}
|
||||||
|
@ -8,9 +8,9 @@ from hmac import HMAC, compare_digest
|
|||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import requests
|
import requests
|
||||||
import redis
|
|
||||||
|
|
||||||
from src.db import DatabaseConnect
|
from src.db import DatabaseConnect
|
||||||
|
from src.ta_redis import TaskHandler
|
||||||
from src.webhook_base import WebhookBase
|
from src.webhook_base import WebhookBase
|
||||||
|
|
||||||
|
|
||||||
@ -544,85 +544,3 @@ class EsVersionSync:
|
|||||||
"""build task for builder"""
|
"""build task for builder"""
|
||||||
task = TaskHandler(self.repo_conf, tag_name=self.expected)
|
task = TaskHandler(self.repo_conf, tag_name=self.expected)
|
||||||
task.create_task("sync_es")
|
task.create_task("sync_es")
|
||||||
|
|
||||||
|
|
||||||
class RedisBase:
|
|
||||||
"""connection base for redis"""
|
|
||||||
|
|
||||||
REDIS_HOST = "redis"
|
|
||||||
REDIS_PORT = 6379
|
|
||||||
NAME_SPACE = "ta:"
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.conn = redis.Redis(host=self.REDIS_HOST, port=self.REDIS_PORT)
|
|
||||||
|
|
||||||
|
|
||||||
class TaskHandler(RedisBase):
|
|
||||||
"""handle buildx task queue"""
|
|
||||||
|
|
||||||
def __init__(self, repo_conf, tag_name=False):
|
|
||||||
super().__init__()
|
|
||||||
self.key = self.NAME_SPACE + "task:buildx"
|
|
||||||
self.repo_conf = repo_conf
|
|
||||||
self.tag_name = tag_name
|
|
||||||
|
|
||||||
def create_task(self, task_name):
|
|
||||||
"""create task"""
|
|
||||||
self.create_queue()
|
|
||||||
self.set_task(task_name)
|
|
||||||
self.set_pub()
|
|
||||||
|
|
||||||
def create_queue(self):
|
|
||||||
"""set initial json object for queue"""
|
|
||||||
if self.conn.execute_command(f"EXISTS {self.key}"):
|
|
||||||
print(f"{self.key} already exists")
|
|
||||||
return
|
|
||||||
|
|
||||||
message = {
|
|
||||||
"created": int(datetime.now().strftime("%s")),
|
|
||||||
"tasks": {}
|
|
||||||
}
|
|
||||||
self.conn.execute_command(
|
|
||||||
"JSON.SET", self.key, ".", json.dumps(message)
|
|
||||||
)
|
|
||||||
|
|
||||||
def set_task(self, task_name):
|
|
||||||
"""publish new task to queue"""
|
|
||||||
|
|
||||||
user = self.repo_conf.get("gh_user")
|
|
||||||
repo = self.repo_conf.get("gh_repo")
|
|
||||||
build_command = self.build_command(task_name)
|
|
||||||
task = {
|
|
||||||
"timestamp": int(datetime.now().strftime("%s")),
|
|
||||||
"clone": f"https://github.com/{user}/{repo}.git",
|
|
||||||
"name": self.repo_conf.get("gh_repo"),
|
|
||||||
"build": build_command,
|
|
||||||
}
|
|
||||||
if task_name == "sync_es":
|
|
||||||
task.update({"clone": False})
|
|
||||||
|
|
||||||
self.conn.json().set(self.key, f".tasks.{repo}", task)
|
|
||||||
|
|
||||||
def build_command(self, task_name):
|
|
||||||
"""return build command"""
|
|
||||||
if not self.tag_name:
|
|
||||||
return self.repo_conf.get(task_name)
|
|
||||||
|
|
||||||
all_commands = self.repo_conf.get(task_name)
|
|
||||||
|
|
||||||
if all(isinstance(i, list) for i in all_commands):
|
|
||||||
to_build_commands = []
|
|
||||||
for command in all_commands:
|
|
||||||
to_build_commands.append(self._replace_version(command))
|
|
||||||
else:
|
|
||||||
to_build_commands = self._replace_version(all_commands)
|
|
||||||
|
|
||||||
return to_build_commands
|
|
||||||
|
|
||||||
def _replace_version(self, command):
|
|
||||||
"""replace version in str"""
|
|
||||||
return [i.replace("$VERSION", self.tag_name) for i in command]
|
|
||||||
|
|
||||||
def set_pub(self):
|
|
||||||
"""set message to pub"""
|
|
||||||
self.conn.publish(self.key, self.repo_conf.get("gh_repo"))
|
|
||||||
|
Loading…
Reference in New Issue
Block a user