mirror of
https://github.com/tubearchivist/tubearchivist-server.git
synced 2024-11-14 16:10:12 +00:00
add docker pull charts
This commit is contained in:
parent
73631d66e6
commit
52b893f4fa
@ -2,6 +2,7 @@ APScheduler==3.10.1
|
||||
beautifulsoup4==4.11.2
|
||||
flask==2.2.3
|
||||
markdown==3.4.1
|
||||
matplotlib==3.7.1
|
||||
pandas==1.5.3
|
||||
psycopg2-binary==2.9.5
|
||||
redis==4.5.1
|
||||
|
161
tubearchivist/web/src/dataset.py
Normal file
161
tubearchivist/web/src/dataset.py
Normal file
@ -0,0 +1,161 @@
|
||||
"""analyze data sets from pg"""
|
||||
|
||||
import matplotlib as mpl
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
|
||||
from src.db import DatabaseConnect
|
||||
|
||||
|
||||
COLORS = {
|
||||
"main-bg": "#00202f",
|
||||
"highlight-bg": "#00293b",
|
||||
"highlight-error": "#990202",
|
||||
"highlight-error-light": "#c44343",
|
||||
"highlight-bg-transparent": "#00293baf",
|
||||
"main-font": "#eeeeee",
|
||||
"accent-font-dark": "#259485",
|
||||
"accent-font-light": "#97d4c8",
|
||||
}
|
||||
|
||||
mpl.rcParams["axes.facecolor"] = COLORS.get("highlight-bg")
|
||||
mpl.rcParams["axes.titlecolor"] = COLORS.get("main-font")
|
||||
mpl.rcParams["axes.titlesize"] = 20
|
||||
mpl.rcParams["figure.facecolor"] = COLORS.get("main-bg")
|
||||
mpl.rcParams["lines.linewidth"] = 2
|
||||
mpl.rcParams["xtick.color"] = COLORS.get("main-font")
|
||||
mpl.rcParams["xtick.labelcolor"] = COLORS.get("main-font")
|
||||
mpl.rcParams["ytick.color"] = COLORS.get("main-font")
|
||||
mpl.rcParams["ytick.labelcolor"] = COLORS.get("main-font")
|
||||
mpl.rcParams["grid.color"] = COLORS.get("main-font")
|
||||
mpl.rcParams["grid.linestyle"] = "--"
|
||||
mpl.rcParams["grid.alpha"] = 0.5
|
||||
|
||||
|
||||
class Pulls:
|
||||
"""analyze docker pulls"""
|
||||
|
||||
query_pulls = """
|
||||
SELECT time_stamp, pulls
|
||||
FROM ta_docker_stats
|
||||
WHERE time_stamp > 1646092800
|
||||
ORDER BY time_stamp DESC;
|
||||
"""
|
||||
|
||||
query_release = """
|
||||
SELECT time_stamp, release_version
|
||||
FROM ta_release
|
||||
WHERE time_stamp > 1646092800
|
||||
ORDER BY time_stamp;
|
||||
"""
|
||||
|
||||
def build_plots(self):
|
||||
"""build plots from dataframe"""
|
||||
pull_rows = self.get_pull_rows()
|
||||
self.cumulative(pull_rows)
|
||||
self.weekly_bar(pull_rows)
|
||||
|
||||
def get_pull_rows(self):
|
||||
"""get rows from pg"""
|
||||
handler = DatabaseConnect()
|
||||
rows = handler.db_execute(self.query_pulls)
|
||||
handler.db_close()
|
||||
|
||||
return rows
|
||||
|
||||
def cumulative(self, rows):
|
||||
"""build cumulative plot"""
|
||||
df = self._load_df(rows)
|
||||
release_rows = self._get_release_rows()
|
||||
self._plot_daily_total(df, release_rows)
|
||||
|
||||
def _get_release_rows(self):
|
||||
"""get release dates and versions"""
|
||||
handler = DatabaseConnect()
|
||||
rows = handler.db_execute(self.query_release)
|
||||
handler.db_close()
|
||||
|
||||
return rows
|
||||
|
||||
def _load_df(self, rows, resample="D"):
|
||||
"""build dataframe from pull rows"""
|
||||
df = pd.DataFrame(rows)
|
||||
df["time_stamp"] = pd.to_datetime(df["time_stamp"], unit="s")
|
||||
df.set_index("time_stamp", inplace=True)
|
||||
|
||||
return df.resample(resample).max()
|
||||
|
||||
def _plot_daily_total(self, df, release_rows):
|
||||
"""create daily total plot"""
|
||||
half_df = self._calc_half_df(df)
|
||||
plt.plot(df, color=COLORS.get("highlight-error"), zorder=3)
|
||||
self._add_releases(release_rows, half_df)
|
||||
plt.title("Cumulative Docker Pulls")
|
||||
plt.savefig("/data/plot-cumulative-docker.png", dpi=300)
|
||||
plt.figure()
|
||||
|
||||
def _calc_half_df(self, df):
|
||||
"""calc the middle of df height"""
|
||||
return (df["pulls"].max() - df["pulls"].min()) // 2 + df["pulls"].min()
|
||||
|
||||
def _add_releases(self, release_rows, half_df):
|
||||
"""add release vert line to plt"""
|
||||
version_colors = {
|
||||
"v0.2.2": COLORS.get("accent-font-light"),
|
||||
"v0.3.1": COLORS.get("accent-font-dark"),
|
||||
"v0.3.3": COLORS.get("accent-font-light"),
|
||||
}
|
||||
release = pd.DataFrame(release_rows)
|
||||
release["time_stamp"] = pd.to_datetime(release["time_stamp"], unit="s")
|
||||
|
||||
for ind in release.index:
|
||||
label = release["release_version"][ind]
|
||||
x_axis = release["time_stamp"][ind]
|
||||
if (match := version_colors.get(label)) is None:
|
||||
match = COLORS.get("main-font")
|
||||
linewidth=0.5
|
||||
else:
|
||||
plt.text(
|
||||
x=x_axis,
|
||||
y=half_df,
|
||||
s=label,
|
||||
color="black",
|
||||
rotation=90,
|
||||
verticalalignment="center",
|
||||
horizontalalignment="center",
|
||||
bbox={"facecolor": match, "edgecolor": 'none'},
|
||||
)
|
||||
linewidth=1
|
||||
|
||||
plt.axvline(
|
||||
x=x_axis,
|
||||
linestyle="--",
|
||||
color=match,
|
||||
linewidth=linewidth,
|
||||
zorder=2,
|
||||
)
|
||||
|
||||
def weekly_bar(self, rows):
|
||||
"""create weekly bar chart"""
|
||||
weekly = self._load_df(rows, resample="W")
|
||||
weekly_diff = weekly.diff().dropna().reset_index()
|
||||
col = [COLORS.get("accent-font-light") for i in weekly_diff["pulls"]]
|
||||
col[-1] = COLORS.get("highlight-error")
|
||||
|
||||
plt.bar(
|
||||
weekly_diff["time_stamp"],
|
||||
weekly_diff["pulls"],
|
||||
width=3,
|
||||
color=col,
|
||||
zorder=1,
|
||||
)
|
||||
plt.grid(zorder=2, axis="y")
|
||||
plt.title("Weekly Docker Pulls")
|
||||
plt.savefig("/data/plot-weekly-docker.png", dpi=300)
|
||||
plt.figure()
|
||||
plt.close()
|
||||
|
||||
|
||||
def run_chart_recreate():
|
||||
"""function to call from scheduler"""
|
||||
Pulls().build_plots()
|
@ -5,6 +5,7 @@ from os import environ
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from flask import Flask, render_template, jsonify, request, redirect
|
||||
from src.api_docker import run_docker_backup
|
||||
from src.dataset import run_chart_recreate
|
||||
from src.versioncheck import VersionCheckCounter, run_version_check_archive
|
||||
from src.webhook_docker import DockerHook
|
||||
from src.webhook_github import GithubBackup, GithubHook
|
||||
@ -29,6 +30,14 @@ scheduler.add_job(
|
||||
minute="0",
|
||||
name="version_backup",
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_chart_recreate,
|
||||
trigger="cron",
|
||||
day="*",
|
||||
hour="2",
|
||||
minute="0",
|
||||
name="chart_recreate"
|
||||
)
|
||||
scheduler.start()
|
||||
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user