feat(forgejo-metrics): refactor line stats fetching to use commit stats endpoint and update caching logic

This commit is contained in:
null 2026-05-22 16:40:09 -05:00
parent 7802400970
commit 2d91325937
3 changed files with 223 additions and 166 deletions

View File

@ -42,32 +42,24 @@ _LINE_STATS_TTL_MISS = 30 # 30 s — retry cadence while Forgejo is still com
async def _bg_fetch_line_stats(
cache_key: str,
# Plain tuples so this task never touches a closed DB session
repos: list[tuple[str, str, str, str | None]], # (owner, repo, base_url, token)
since_ts: float,
since_iso: str, # ISO-8601 date string for the commits?since= filter
) -> None:
"""Background task: fetch per-repo contributor stats and cache the totals."""
"""Background task: sum commit line stats across all tracked repos and cache."""
async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int, bool]:
async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int]:
try:
async with ForgejoAPIClient(base_url=base_url, token=token) as client:
contributors, has_data = await client.get_contributor_stats(owner, repo)
adds = dels = 0
for contributor in contributors:
for week in contributor.get("weeks", []):
if (week.get("w") or 0) >= since_ts:
adds += week.get("a", 0) or 0
dels += week.get("d", 0) or 0
return adds, dels, has_data
return await client.get_commit_line_stats_since(owner, repo, since_iso)
except Exception:
return 0, 0, False
return 0, 0
try:
results = await asyncio.gather(*[_one(o, r, bu, tok) for o, r, bu, tok in repos])
total_adds = sum(a for a, _, _ in results)
total_dels = sum(d for _, d, _ in results)
has_data = bool(results) and all(ok for _, _, ok in results)
_line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, has_data)
total_adds = sum(a for a, _ in results)
total_dels = sum(d for _, d in results)
# Always mark has_data=True — the commits endpoint is reliable
_line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, True)
finally:
_line_stats_fetching.discard(cache_key)
@ -449,7 +441,7 @@ async def get_forgejo_heatmap(
# Line stats — served from cache; background task refreshes when stale.
# Extract plain values NOW while the DB session is still open.
since_ts = since.timestamp()
since_iso = since.strftime("%Y-%m-%dT%H:%M:%SZ")
cache_key = str(ctx.organization.id)
cached = _line_stats_cache.get(cache_key)
now = _time.monotonic()
@ -471,7 +463,7 @@ async def get_forgejo_heatmap(
_line_stats_fetching.add(cache_key)
asyncio.create_task(
_bg_fetch_line_stats(cache_key, repo_tuples, since_ts),
_bg_fetch_line_stats(cache_key, repo_tuples, since_iso),
name=f"line-stats-{cache_key}",
)

View File

@ -368,28 +368,38 @@ class ForgejoAPIClient:
commits = data if isinstance(data, list) else data.get("commits") or data.get("data") or []
return commits[0] if commits else None
async def get_contributor_stats(self, owner: str, repo: str) -> tuple[list[dict], bool]:
"""Fetch per-contributor weekly stats for a repository.
async def get_commit_line_stats_since(
self, owner: str, repo: str, since_iso: str
) -> tuple[int, int]:
"""Sum additions and deletions across all commits since ``since_iso``.
Returns (contributors, has_data). Forgejo returns HTTP 202 while it
computes stats that is not an error, it just means the caller should
retry later. has_data=False signals the 202 case; the application layer
uses a short retry TTL so it probes again within ~30 s.
Each contributor dict has a ``weeks`` array with ``w`` (Unix timestamp),
``a`` (additions), and ``d`` (deletions).
Uses ``GET /repos/{owner}/{repo}/commits?since=&stat=true`` which
returns per-commit ``stats`` objects and is available on all Forgejo/
Gitea versions. Returns ``(total_additions, total_deletions)``.
"""
client = await self._get_client()
response = await client.get(
f"/api/v1/repos/{owner}/{repo}/stats/contributors"
)
if response.status_code == 202:
return [], False # still computing — caller will retry shortly
if response.status_code == 404:
return [], True # repo exists but no commits; treat as data-present
response.raise_for_status()
data = response.json()
return (data if isinstance(data, list) else []), True
total_adds = total_dels = 0
page = 1
while True:
response = await client.get(
f"/api/v1/repos/{owner}/{repo}/commits",
params={"since": since_iso, "limit": 50, "page": page, "stat": "true"},
)
if response.status_code in (404, 409):
break # empty or non-existent repo
response.raise_for_status()
data = response.json()
commits = data if isinstance(data, list) else []
if not commits:
break
for commit in commits:
s = commit.get("stats") or {}
total_adds += int(s.get("additions") or 0)
total_dels += int(s.get("deletions") or 0)
if len(commits) < 50:
break # last page
page += 1
return total_adds, total_dels
def get_forgejo_client(

View File

@ -13,36 +13,25 @@ interface ForgejoHeatmapProps {
isLoading?: boolean;
}
const CELL = 16;
const GAP = 4;
const STRIDE = CELL + GAP;
const WEEKS = 27;
const LEFT = 32;
const TOP = 20;
// Chart layout
const VW = 700; // viewBox width
const VH = 160; // viewBox height
const PAD_L = 8;
const PAD_R = 8;
const PAD_T = 12;
const PAD_B = 28; // room for month labels
const CW = VW - PAD_L - PAD_R; // chart width
const CH = VH - PAD_T - PAD_B; // chart height
const DAYS = 183;
const MONTHS = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"];
// All fixed rgba — no CSS vars inside SVG fill so every browser renders them
const LEVEL_FILL = [
"rgba(139,92,246,0.08)", // 0 empty — barely-there grid
"rgba(139,92,246,0.30)", // 1
"rgba(139,92,246,0.54)", // 2
"rgba(139,92,246,0.77)", // 3
"rgba(139,92,246,1.00)", // 4 full violet-500
];
const VIOLET = "rgba(139,92,246,1)";
const GREEN = "rgba(52,211,153,1)";
const RED = "rgba(248,113,113,1)";
const LABEL_CLR = "rgba(255,255,255,0.70)"; // white labels
function toLevel(count: number): number {
if (count === 0) return 0;
if (count <= 2) return 1;
if (count <= 5) return 2;
if (count <= 9) return 3;
return 4;
}
const VIOLET = "rgba(139,92,246,1)";
const GREEN = "rgba(52,211,153,1)";
const RED = "rgba(248,113,113,1)";
const WHITE70 = "rgba(255,255,255,0.70)";
const WHITE50 = "rgba(255,255,255,0.50)";
const WHITE15 = "rgba(255,255,255,0.15)";
function isoDate(d: Date): string {
return [
@ -59,132 +48,206 @@ function fmtLines(n: number): string {
}
function fmtRelative(iso: string): string {
const diffMs = Date.now() - new Date(iso).getTime();
const s = Math.floor(diffMs / 1000);
if (s < 60) return "just now";
const s = Math.floor((Date.now() - new Date(iso).getTime()) / 1000);
if (s < 60) return "just now";
const m = Math.floor(s / 60);
if (m < 60) return `${m}m ago`;
if (m < 60) return `${m}m ago`;
const h = Math.floor(m / 60);
if (h < 24) return `${h}h ago`;
if (h < 24) return `${h}h ago`;
return `${Math.floor(h / 24)}d ago`;
}
export function ForgejoHeatmap({
days,
maxCount: _maxCount,
maxCount,
totalAdditions = 0,
totalDeletions = 0,
hasLineStats = false,
lastPush = null,
isLoading = false,
}: ForgejoHeatmapProps) {
const { weeks, monthLabels } = useMemo(() => {
const data = new Map(days.map((d) => [d.date, d.count]));
const { points, monthLabels, gridLines, peak } = useMemo(() => {
const lookup = new Map(days.map((d) => [d.date, d.count]));
// Build a dense 183-day array ending today
const today = new Date();
today.setHours(0, 0, 0, 0);
const start = new Date(today);
start.setDate(start.getDate() - start.getDay());
start.setDate(start.getDate() - (WEEKS - 1) * 7);
type Cell = { date: string; count: number; future: boolean };
const builtWeeks: Cell[][] = [];
const monthLabelList: { weekIdx: number; label: string }[] = [];
let lastMonth = -1;
const cur = new Date(start);
for (let w = 0; w < WEEKS; w++) {
const week: Cell[] = [];
for (let d = 0; d < 7; d++) {
const dateStr = isoDate(cur);
const month = cur.getMonth();
if (d === 0 && month !== lastMonth) {
monthLabelList.push({ weekIdx: w, label: MONTHS[month] });
lastMonth = month;
}
week.push({ date: dateStr, count: data.get(dateStr) ?? 0, future: cur > today });
cur.setDate(cur.getDate() + 1);
}
builtWeeks.push(week);
const series: { date: string; count: number }[] = [];
for (let i = DAYS - 1; i >= 0; i--) {
const d = new Date(today);
d.setDate(d.getDate() - i);
const key = isoDate(d);
series.push({ date: key, count: lookup.get(key) ?? 0 });
}
return { weeks: builtWeeks, monthLabels: monthLabelList };
}, [days]);
const localMax = Math.max(...series.map((p) => p.count), 1);
const svgW = LEFT + WEEKS * STRIDE;
const svgH = TOP + 7 * STRIDE - GAP;
// Map to SVG coords
const pts = series.map((p, i) => ({
x: PAD_L + (i / (DAYS - 1)) * CW,
y: PAD_T + CH - (p.count / localMax) * CH,
date: p.date,
count: p.count,
}));
// Month label positions
const labels: { x: number; label: string }[] = [];
let lastMonth = -1;
series.forEach((p, i) => {
const m = new Date(p.date).getMonth();
if (m !== lastMonth) {
labels.push({ x: PAD_L + (i / (DAYS - 1)) * CW, label: MONTHS[m] });
lastMonth = m;
}
});
// Horizontal grid lines at 25 / 50 / 75 % of chart height
const grid = [0.25, 0.5, 0.75].map((f) => PAD_T + CH * (1 - f));
// Peak point (highest count)
const peak = pts.reduce((best, p) => (p.count > best.count ? p : best), pts[0]);
return { points: pts, monthLabels: labels, gridLines: grid, peak };
}, [days, maxCount]);
// Build SVG path strings
const lineD = useMemo(() => {
if (!points.length) return "";
return points.map((p, i) => `${i === 0 ? "M" : "L"}${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" ");
}, [points]);
const areaD = useMemo(() => {
if (!points.length) return "";
const bottom = PAD_T + CH;
return (
`M${PAD_L},${bottom} ` +
points.map((p) => `L${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" ") +
` L${(PAD_L + CW).toFixed(1)},${bottom} Z`
);
}, [points]);
const totalEvents = days.reduce((sum, d) => sum + d.count, 0);
if (isLoading) {
return (
<div className="space-y-4">
<div className="animate-pulse rounded" style={{ width: svgW, height: svgH, background: "rgba(139,92,246,0.08)" }} />
<div className="mx-auto h-8 w-64 animate-pulse rounded-full" style={{ background: "rgba(139,92,246,0.08)" }} />
<div className="w-full animate-pulse rounded-lg" style={{ height: VH, background: "rgba(139,92,246,0.08)" }} />
<div className="mx-auto h-6 w-64 animate-pulse rounded" style={{ background: "rgba(139,92,246,0.08)" }} />
</div>
);
}
const totalEvents = days.reduce((sum, d) => sum + d.count, 0);
return (
<div className="space-y-4">
{/* ── Heatmap ─────────────────────────────────────────────── */}
<div className="flex justify-center" style={{ overflowX: "auto" }}>
<svg width={svgW} height={svgH} aria-label="Issue activity heatmap" style={{ display: "block" }}>
{/* ── Area chart ──────────────────────────────────────────── */}
<svg
viewBox={`0 0 ${VW} ${VH}`}
width="100%"
aria-label="Git activity graph"
style={{ display: "block", overflow: "visible" }}
>
<defs>
<linearGradient id="git-area-gradient" x1="0" y1="0" x2="0" y2="1">
<stop offset="0%" stopColor="rgba(139,92,246,0.45)" />
<stop offset="100%" stopColor="rgba(139,92,246,0.02)" />
</linearGradient>
<clipPath id="git-chart-clip">
<rect x={PAD_L} y={PAD_T} width={CW} height={CH} />
</clipPath>
</defs>
{/* Month labels */}
{monthLabels.map(({ weekIdx, label }) => (
<text key={`m-${weekIdx}`} x={LEFT + weekIdx * STRIDE} y={11} fontSize={10} style={{ fill: LABEL_CLR }}>
{label}
</text>
))}
{/* Day labels */}
{(["Mon", "Wed", "Fri"] as const).map((label, i) => (
<text key={label} x={0} y={TOP + (i * 2 + 1) * STRIDE + CELL - 2} fontSize={10} style={{ fill: LABEL_CLR }}>
{label}
</text>
))}
{/* Cells — style.fill so rgba resolves correctly in SVG */}
{weeks.map((week, wi) =>
week.map((cell, di) =>
cell.future ? null : (
<rect
key={cell.date}
x={LEFT + wi * STRIDE}
y={TOP + di * STRIDE}
width={CELL}
height={CELL}
rx={3}
style={{ fill: LEVEL_FILL[toLevel(cell.count)] }}
>
<title>
{cell.date}
{cell.count > 0 ? `: ${cell.count} event${cell.count !== 1 ? "s" : ""}` : ": no activity"}
</title>
</rect>
)
)
)}
</svg>
</div>
{/* ── Legend ──────────────────────────────────────────────── */}
<div className="flex items-center justify-center gap-1.5 text-xs" style={{ color: LABEL_CLR }}>
<span>Less</span>
{LEVEL_FILL.map((fill, i) => (
<div key={i} style={{ width: CELL, height: CELL, borderRadius: 3, background: fill, flexShrink: 0 }} />
{/* Grid lines */}
{gridLines.map((y, i) => (
<line
key={i}
x1={PAD_L} y1={y} x2={PAD_L + CW} y2={y}
style={{ stroke: WHITE15, strokeWidth: 1 }}
strokeDasharray="3 4"
/>
))}
<span>More</span>
</div>
{/* Filled area */}
<path
d={areaD}
style={{ fill: "url(#git-area-gradient)" }}
clipPath="url(#git-chart-clip)"
/>
{/* Line */}
<path
d={lineD}
style={{ fill: "none", stroke: VIOLET, strokeWidth: 1.8, strokeLinejoin: "round", strokeLinecap: "round" }}
clipPath="url(#git-chart-clip)"
/>
{/* Peak dot + label */}
{peak && peak.count > 0 && (
<>
<circle cx={peak.x} cy={peak.y} r={4} style={{ fill: VIOLET }} />
<circle cx={peak.x} cy={peak.y} r={7} style={{ fill: "rgba(139,92,246,0.20)" }} />
<text
x={Math.min(Math.max(peak.x, PAD_L + 16), PAD_L + CW - 16)}
y={peak.y - 12}
fontSize={10}
textAnchor="middle"
style={{ fill: WHITE70 }}
>
{peak.count}
</text>
</>
)}
{/* Today dot */}
{points.length > 0 && (
<circle
cx={points[points.length - 1].x}
cy={points[points.length - 1].y}
r={3}
style={{ fill: VIOLET }}
/>
)}
{/* X-axis baseline */}
<line
x1={PAD_L} y1={PAD_T + CH} x2={PAD_L + CW} y2={PAD_T + CH}
style={{ stroke: WHITE15, strokeWidth: 1 }}
/>
{/* Month labels */}
{monthLabels.map(({ x, label }, i) => (
<text
key={i}
x={x}
y={VH - 8}
fontSize={10}
style={{ fill: WHITE70 }}
>
{label}
</text>
))}
{/* Invisible hit targets for per-day tooltip */}
{points.map((p) => (
<rect
key={p.date}
x={p.x - CW / (DAYS * 2)}
y={PAD_T}
width={CW / DAYS}
height={CH}
style={{ fill: "transparent", cursor: "crosshair" }}
>
<title>{p.date}{p.count > 0 ? `: ${p.count} event${p.count !== 1 ? "s" : ""}` : ": no activity"}</title>
</rect>
))}
</svg>
{/* ── Contributions summary ────────────────────────────────── */}
<p className="text-center">
<span className="text-2xl font-bold tabular-nums" style={{ color: VIOLET }}>
{totalEvents.toLocaleString()}
</span>
<span className="ml-1.5 text-sm" style={{ color: "rgba(255,255,255,0.70)" }}>
<span className="ml-1.5 text-sm" style={{ color: WHITE70 }}>
contributions across all tracked repositories in the last 6 months
</span>
</p>
@ -195,7 +258,6 @@ export function ForgejoHeatmap({
className="mx-auto flex max-w-lg items-start gap-3 rounded-lg px-4 py-2.5"
style={{ background: "rgba(139,92,246,0.08)", border: "1px solid rgba(139,92,246,0.18)" }}
>
{/* commit hash pill */}
<span
className="mt-0.5 shrink-0 rounded px-1.5 py-0.5 font-mono text-[11px] font-semibold"
style={{ background: "rgba(139,92,246,0.20)", color: VIOLET }}
@ -206,13 +268,10 @@ export function ForgejoHeatmap({
<p className="truncate text-sm font-medium" style={{ color: "rgba(255,255,255,0.90)" }}>
{lastPush.message}
</p>
<p className="mt-0.5 text-xs" style={{ color: "rgba(255,255,255,0.50)" }}>
<p className="mt-0.5 text-xs" style={{ color: WHITE50 }}>
{lastPush.author} pushed to{" "}
<span style={{ color: "rgba(139,92,246,0.90)" }}>{lastPush.branch}</span>
{" · "}
{lastPush.repo}
{" · "}
{fmtRelative(lastPush.pushed_at)}
{" · "}{lastPush.repo}{" · "}{fmtRelative(lastPush.pushed_at)}
</p>
</div>
</div>
@ -225,22 +284,18 @@ export function ForgejoHeatmap({
<span className="text-2xl font-bold tabular-nums" style={{ color: GREEN }}>
+{fmtLines(totalAdditions)}
</span>
<span className="text-xs font-medium" style={{ color: "rgba(52,211,153,0.70)" }}>
lines added
</span>
<span className="text-xs font-medium" style={{ color: "rgba(52,211,153,0.70)" }}>lines added</span>
</div>
<div style={{ width: 1, height: 40, background: "rgba(139,92,246,0.20)" }} />
<div className="flex flex-col items-center gap-0.5">
<span className="text-2xl font-bold tabular-nums" style={{ color: RED }}>
-{fmtLines(totalDeletions)}
</span>
<span className="text-xs font-medium" style={{ color: "rgba(248,113,113,0.70)" }}>
lines removed
</span>
<span className="text-xs font-medium" style={{ color: "rgba(248,113,113,0.70)" }}>lines removed</span>
</div>
</div>
) : (
<p className="text-center text-xs" style={{ color: LABEL_CLR, opacity: 0.6 }}>
<p className="text-center text-xs" style={{ color: WHITE70, opacity: 0.6 }}>
Line stats syncing with Forgejo will appear on next refresh
</p>
)}