add docker pull charts

This commit is contained in:
simon 2023-03-07 13:22:19 +07:00
parent 73631d66e6
commit 52b893f4fa
No known key found for this signature in database
GPG Key ID: 2C15AA5E89985DD4
3 changed files with 171 additions and 0 deletions

View File

@ -2,6 +2,7 @@ APScheduler==3.10.1
beautifulsoup4==4.11.2 beautifulsoup4==4.11.2
flask==2.2.3 flask==2.2.3
markdown==3.4.1 markdown==3.4.1
matplotlib==3.7.1
pandas==1.5.3 pandas==1.5.3
psycopg2-binary==2.9.5 psycopg2-binary==2.9.5
redis==4.5.1 redis==4.5.1

View File

@ -0,0 +1,161 @@
"""analyze data sets from pg"""
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
from src.db import DatabaseConnect
COLORS = {
"main-bg": "#00202f",
"highlight-bg": "#00293b",
"highlight-error": "#990202",
"highlight-error-light": "#c44343",
"highlight-bg-transparent": "#00293baf",
"main-font": "#eeeeee",
"accent-font-dark": "#259485",
"accent-font-light": "#97d4c8",
}
mpl.rcParams["axes.facecolor"] = COLORS.get("highlight-bg")
mpl.rcParams["axes.titlecolor"] = COLORS.get("main-font")
mpl.rcParams["axes.titlesize"] = 20
mpl.rcParams["figure.facecolor"] = COLORS.get("main-bg")
mpl.rcParams["lines.linewidth"] = 2
mpl.rcParams["xtick.color"] = COLORS.get("main-font")
mpl.rcParams["xtick.labelcolor"] = COLORS.get("main-font")
mpl.rcParams["ytick.color"] = COLORS.get("main-font")
mpl.rcParams["ytick.labelcolor"] = COLORS.get("main-font")
mpl.rcParams["grid.color"] = COLORS.get("main-font")
mpl.rcParams["grid.linestyle"] = "--"
mpl.rcParams["grid.alpha"] = 0.5
class Pulls:
"""analyze docker pulls"""
query_pulls = """
SELECT time_stamp, pulls
FROM ta_docker_stats
WHERE time_stamp > 1646092800
ORDER BY time_stamp DESC;
"""
query_release = """
SELECT time_stamp, release_version
FROM ta_release
WHERE time_stamp > 1646092800
ORDER BY time_stamp;
"""
def build_plots(self):
"""build plots from dataframe"""
pull_rows = self.get_pull_rows()
self.cumulative(pull_rows)
self.weekly_bar(pull_rows)
def get_pull_rows(self):
"""get rows from pg"""
handler = DatabaseConnect()
rows = handler.db_execute(self.query_pulls)
handler.db_close()
return rows
def cumulative(self, rows):
"""build cumulative plot"""
df = self._load_df(rows)
release_rows = self._get_release_rows()
self._plot_daily_total(df, release_rows)
def _get_release_rows(self):
"""get release dates and versions"""
handler = DatabaseConnect()
rows = handler.db_execute(self.query_release)
handler.db_close()
return rows
def _load_df(self, rows, resample="D"):
"""build dataframe from pull rows"""
df = pd.DataFrame(rows)
df["time_stamp"] = pd.to_datetime(df["time_stamp"], unit="s")
df.set_index("time_stamp", inplace=True)
return df.resample(resample).max()
def _plot_daily_total(self, df, release_rows):
"""create daily total plot"""
half_df = self._calc_half_df(df)
plt.plot(df, color=COLORS.get("highlight-error"), zorder=3)
self._add_releases(release_rows, half_df)
plt.title("Cumulative Docker Pulls")
plt.savefig("/data/plot-cumulative-docker.png", dpi=300)
plt.figure()
def _calc_half_df(self, df):
"""calc the middle of df height"""
return (df["pulls"].max() - df["pulls"].min()) // 2 + df["pulls"].min()
def _add_releases(self, release_rows, half_df):
"""add release vert line to plt"""
version_colors = {
"v0.2.2": COLORS.get("accent-font-light"),
"v0.3.1": COLORS.get("accent-font-dark"),
"v0.3.3": COLORS.get("accent-font-light"),
}
release = pd.DataFrame(release_rows)
release["time_stamp"] = pd.to_datetime(release["time_stamp"], unit="s")
for ind in release.index:
label = release["release_version"][ind]
x_axis = release["time_stamp"][ind]
if (match := version_colors.get(label)) is None:
match = COLORS.get("main-font")
linewidth=0.5
else:
plt.text(
x=x_axis,
y=half_df,
s=label,
color="black",
rotation=90,
verticalalignment="center",
horizontalalignment="center",
bbox={"facecolor": match, "edgecolor": 'none'},
)
linewidth=1
plt.axvline(
x=x_axis,
linestyle="--",
color=match,
linewidth=linewidth,
zorder=2,
)
def weekly_bar(self, rows):
"""create weekly bar chart"""
weekly = self._load_df(rows, resample="W")
weekly_diff = weekly.diff().dropna().reset_index()
col = [COLORS.get("accent-font-light") for i in weekly_diff["pulls"]]
col[-1] = COLORS.get("highlight-error")
plt.bar(
weekly_diff["time_stamp"],
weekly_diff["pulls"],
width=3,
color=col,
zorder=1,
)
plt.grid(zorder=2, axis="y")
plt.title("Weekly Docker Pulls")
plt.savefig("/data/plot-weekly-docker.png", dpi=300)
plt.figure()
plt.close()
def run_chart_recreate():
"""function to call from scheduler"""
Pulls().build_plots()

View File

@ -5,6 +5,7 @@ from os import environ
from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask, render_template, jsonify, request, redirect from flask import Flask, render_template, jsonify, request, redirect
from src.api_docker import run_docker_backup from src.api_docker import run_docker_backup
from src.dataset import run_chart_recreate
from src.versioncheck import VersionCheckCounter, run_version_check_archive from src.versioncheck import VersionCheckCounter, run_version_check_archive
from src.webhook_docker import DockerHook from src.webhook_docker import DockerHook
from src.webhook_github import GithubBackup, GithubHook from src.webhook_github import GithubBackup, GithubHook
@ -29,6 +30,14 @@ scheduler.add_job(
minute="0", minute="0",
name="version_backup", name="version_backup",
) )
scheduler.add_job(
run_chart_recreate,
trigger="cron",
day="*",
hour="2",
minute="0",
name="chart_recreate"
)
scheduler.start() scheduler.start()