From 2d91325937d175d352ba534c2373de6473ddc2a5 Mon Sep 17 00:00:00 2001 From: null Date: Fri, 22 May 2026 16:40:09 -0500 Subject: [PATCH] feat(forgejo-metrics): refactor line stats fetching to use commit stats endpoint and update caching logic --- backend/app/api/forgejo_metrics.py | 30 +- backend/app/services/forgejo_client.py | 48 +-- .../src/components/git/ForgejoHeatmap.tsx | 311 +++++++++++------- 3 files changed, 223 insertions(+), 166 deletions(-) diff --git a/backend/app/api/forgejo_metrics.py b/backend/app/api/forgejo_metrics.py index 1ec375c..d4e413e 100644 --- a/backend/app/api/forgejo_metrics.py +++ b/backend/app/api/forgejo_metrics.py @@ -42,32 +42,24 @@ _LINE_STATS_TTL_MISS = 30 # 30 s — retry cadence while Forgejo is still com async def _bg_fetch_line_stats( cache_key: str, - # Plain tuples so this task never touches a closed DB session repos: list[tuple[str, str, str, str | None]], # (owner, repo, base_url, token) - since_ts: float, + since_iso: str, # ISO-8601 date string for the commits?since= filter ) -> None: - """Background task: fetch per-repo contributor stats and cache the totals.""" + """Background task: sum commit line stats across all tracked repos and cache.""" - async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int, bool]: + async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int]: try: async with ForgejoAPIClient(base_url=base_url, token=token) as client: - contributors, has_data = await client.get_contributor_stats(owner, repo) - adds = dels = 0 - for contributor in contributors: - for week in contributor.get("weeks", []): - if (week.get("w") or 0) >= since_ts: - adds += week.get("a", 0) or 0 - dels += week.get("d", 0) or 0 - return adds, dels, has_data + return await client.get_commit_line_stats_since(owner, repo, since_iso) except Exception: - return 0, 0, False + return 0, 0 try: results = await asyncio.gather(*[_one(o, r, bu, tok) for o, r, bu, tok in repos]) - total_adds = sum(a for a, _, _ in results) - total_dels = sum(d for _, d, _ in results) - has_data = bool(results) and all(ok for _, _, ok in results) - _line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, has_data) + total_adds = sum(a for a, _ in results) + total_dels = sum(d for _, d in results) + # Always mark has_data=True — the commits endpoint is reliable + _line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, True) finally: _line_stats_fetching.discard(cache_key) @@ -449,7 +441,7 @@ async def get_forgejo_heatmap( # Line stats — served from cache; background task refreshes when stale. # Extract plain values NOW while the DB session is still open. - since_ts = since.timestamp() + since_iso = since.strftime("%Y-%m-%dT%H:%M:%SZ") cache_key = str(ctx.organization.id) cached = _line_stats_cache.get(cache_key) now = _time.monotonic() @@ -471,7 +463,7 @@ async def get_forgejo_heatmap( _line_stats_fetching.add(cache_key) asyncio.create_task( - _bg_fetch_line_stats(cache_key, repo_tuples, since_ts), + _bg_fetch_line_stats(cache_key, repo_tuples, since_iso), name=f"line-stats-{cache_key}", ) diff --git a/backend/app/services/forgejo_client.py b/backend/app/services/forgejo_client.py index c453133..5038f04 100644 --- a/backend/app/services/forgejo_client.py +++ b/backend/app/services/forgejo_client.py @@ -368,28 +368,38 @@ class ForgejoAPIClient: commits = data if isinstance(data, list) else data.get("commits") or data.get("data") or [] return commits[0] if commits else None - async def get_contributor_stats(self, owner: str, repo: str) -> tuple[list[dict], bool]: - """Fetch per-contributor weekly stats for a repository. + async def get_commit_line_stats_since( + self, owner: str, repo: str, since_iso: str + ) -> tuple[int, int]: + """Sum additions and deletions across all commits since ``since_iso``. - Returns (contributors, has_data). Forgejo returns HTTP 202 while it - computes stats — that is not an error, it just means the caller should - retry later. has_data=False signals the 202 case; the application layer - uses a short retry TTL so it probes again within ~30 s. - - Each contributor dict has a ``weeks`` array with ``w`` (Unix timestamp), - ``a`` (additions), and ``d`` (deletions). + Uses ``GET /repos/{owner}/{repo}/commits?since=…&stat=true`` which + returns per-commit ``stats`` objects and is available on all Forgejo/ + Gitea versions. Returns ``(total_additions, total_deletions)``. """ client = await self._get_client() - response = await client.get( - f"/api/v1/repos/{owner}/{repo}/stats/contributors" - ) - if response.status_code == 202: - return [], False # still computing — caller will retry shortly - if response.status_code == 404: - return [], True # repo exists but no commits; treat as data-present - response.raise_for_status() - data = response.json() - return (data if isinstance(data, list) else []), True + total_adds = total_dels = 0 + page = 1 + while True: + response = await client.get( + f"/api/v1/repos/{owner}/{repo}/commits", + params={"since": since_iso, "limit": 50, "page": page, "stat": "true"}, + ) + if response.status_code in (404, 409): + break # empty or non-existent repo + response.raise_for_status() + data = response.json() + commits = data if isinstance(data, list) else [] + if not commits: + break + for commit in commits: + s = commit.get("stats") or {} + total_adds += int(s.get("additions") or 0) + total_dels += int(s.get("deletions") or 0) + if len(commits) < 50: + break # last page + page += 1 + return total_adds, total_dels def get_forgejo_client( diff --git a/frontend/src/components/git/ForgejoHeatmap.tsx b/frontend/src/components/git/ForgejoHeatmap.tsx index d97355b..093b8d9 100644 --- a/frontend/src/components/git/ForgejoHeatmap.tsx +++ b/frontend/src/components/git/ForgejoHeatmap.tsx @@ -13,36 +13,25 @@ interface ForgejoHeatmapProps { isLoading?: boolean; } -const CELL = 16; -const GAP = 4; -const STRIDE = CELL + GAP; -const WEEKS = 27; -const LEFT = 32; -const TOP = 20; +// Chart layout +const VW = 700; // viewBox width +const VH = 160; // viewBox height +const PAD_L = 8; +const PAD_R = 8; +const PAD_T = 12; +const PAD_B = 28; // room for month labels +const CW = VW - PAD_L - PAD_R; // chart width +const CH = VH - PAD_T - PAD_B; // chart height +const DAYS = 183; const MONTHS = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]; -// All fixed rgba — no CSS vars inside SVG fill so every browser renders them -const LEVEL_FILL = [ - "rgba(139,92,246,0.08)", // 0 empty — barely-there grid - "rgba(139,92,246,0.30)", // 1 - "rgba(139,92,246,0.54)", // 2 - "rgba(139,92,246,0.77)", // 3 - "rgba(139,92,246,1.00)", // 4 full violet-500 -]; - -const VIOLET = "rgba(139,92,246,1)"; -const GREEN = "rgba(52,211,153,1)"; -const RED = "rgba(248,113,113,1)"; -const LABEL_CLR = "rgba(255,255,255,0.70)"; // white labels - -function toLevel(count: number): number { - if (count === 0) return 0; - if (count <= 2) return 1; - if (count <= 5) return 2; - if (count <= 9) return 3; - return 4; -} +const VIOLET = "rgba(139,92,246,1)"; +const GREEN = "rgba(52,211,153,1)"; +const RED = "rgba(248,113,113,1)"; +const WHITE70 = "rgba(255,255,255,0.70)"; +const WHITE50 = "rgba(255,255,255,0.50)"; +const WHITE15 = "rgba(255,255,255,0.15)"; function isoDate(d: Date): string { return [ @@ -59,132 +48,206 @@ function fmtLines(n: number): string { } function fmtRelative(iso: string): string { - const diffMs = Date.now() - new Date(iso).getTime(); - const s = Math.floor(diffMs / 1000); - if (s < 60) return "just now"; + const s = Math.floor((Date.now() - new Date(iso).getTime()) / 1000); + if (s < 60) return "just now"; const m = Math.floor(s / 60); - if (m < 60) return `${m}m ago`; + if (m < 60) return `${m}m ago`; const h = Math.floor(m / 60); - if (h < 24) return `${h}h ago`; + if (h < 24) return `${h}h ago`; return `${Math.floor(h / 24)}d ago`; } export function ForgejoHeatmap({ days, - maxCount: _maxCount, + maxCount, totalAdditions = 0, totalDeletions = 0, hasLineStats = false, lastPush = null, isLoading = false, }: ForgejoHeatmapProps) { - const { weeks, monthLabels } = useMemo(() => { - const data = new Map(days.map((d) => [d.date, d.count])); + const { points, monthLabels, gridLines, peak } = useMemo(() => { + const lookup = new Map(days.map((d) => [d.date, d.count])); + // Build a dense 183-day array ending today const today = new Date(); today.setHours(0, 0, 0, 0); - const start = new Date(today); - start.setDate(start.getDate() - start.getDay()); - start.setDate(start.getDate() - (WEEKS - 1) * 7); - - type Cell = { date: string; count: number; future: boolean }; - const builtWeeks: Cell[][] = []; - const monthLabelList: { weekIdx: number; label: string }[] = []; - let lastMonth = -1; - const cur = new Date(start); - - for (let w = 0; w < WEEKS; w++) { - const week: Cell[] = []; - for (let d = 0; d < 7; d++) { - const dateStr = isoDate(cur); - const month = cur.getMonth(); - if (d === 0 && month !== lastMonth) { - monthLabelList.push({ weekIdx: w, label: MONTHS[month] }); - lastMonth = month; - } - week.push({ date: dateStr, count: data.get(dateStr) ?? 0, future: cur > today }); - cur.setDate(cur.getDate() + 1); - } - builtWeeks.push(week); + const series: { date: string; count: number }[] = []; + for (let i = DAYS - 1; i >= 0; i--) { + const d = new Date(today); + d.setDate(d.getDate() - i); + const key = isoDate(d); + series.push({ date: key, count: lookup.get(key) ?? 0 }); } - return { weeks: builtWeeks, monthLabels: monthLabelList }; - }, [days]); + const localMax = Math.max(...series.map((p) => p.count), 1); - const svgW = LEFT + WEEKS * STRIDE; - const svgH = TOP + 7 * STRIDE - GAP; + // Map to SVG coords + const pts = series.map((p, i) => ({ + x: PAD_L + (i / (DAYS - 1)) * CW, + y: PAD_T + CH - (p.count / localMax) * CH, + date: p.date, + count: p.count, + })); + + // Month label positions + const labels: { x: number; label: string }[] = []; + let lastMonth = -1; + series.forEach((p, i) => { + const m = new Date(p.date).getMonth(); + if (m !== lastMonth) { + labels.push({ x: PAD_L + (i / (DAYS - 1)) * CW, label: MONTHS[m] }); + lastMonth = m; + } + }); + + // Horizontal grid lines at 25 / 50 / 75 % of chart height + const grid = [0.25, 0.5, 0.75].map((f) => PAD_T + CH * (1 - f)); + + // Peak point (highest count) + const peak = pts.reduce((best, p) => (p.count > best.count ? p : best), pts[0]); + + return { points: pts, monthLabels: labels, gridLines: grid, peak }; + }, [days, maxCount]); + + // Build SVG path strings + const lineD = useMemo(() => { + if (!points.length) return ""; + return points.map((p, i) => `${i === 0 ? "M" : "L"}${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" "); + }, [points]); + + const areaD = useMemo(() => { + if (!points.length) return ""; + const bottom = PAD_T + CH; + return ( + `M${PAD_L},${bottom} ` + + points.map((p) => `L${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" ") + + ` L${(PAD_L + CW).toFixed(1)},${bottom} Z` + ); + }, [points]); + + const totalEvents = days.reduce((sum, d) => sum + d.count, 0); if (isLoading) { return (
-
-
+
+
); } - const totalEvents = days.reduce((sum, d) => sum + d.count, 0); - return (
- {/* ── Heatmap ─────────────────────────────────────────────── */} -
- + {/* ── Area chart ──────────────────────────────────────────── */} + + + + + + + + + + - {/* Month labels */} - {monthLabels.map(({ weekIdx, label }) => ( - - {label} - - ))} - - {/* Day labels */} - {(["Mon", "Wed", "Fri"] as const).map((label, i) => ( - - {label} - - ))} - - {/* Cells — style.fill so rgba resolves correctly in SVG */} - {weeks.map((week, wi) => - week.map((cell, di) => - cell.future ? null : ( - - - {cell.date} - {cell.count > 0 ? `: ${cell.count} event${cell.count !== 1 ? "s" : ""}` : ": no activity"} - - - ) - ) - )} - -
- - {/* ── Legend ──────────────────────────────────────────────── */} -
- Less - {LEVEL_FILL.map((fill, i) => ( -
+ {/* Grid lines */} + {gridLines.map((y, i) => ( + ))} - More -
+ + {/* Filled area */} + + + {/* Line */} + + + {/* Peak dot + label */} + {peak && peak.count > 0 && ( + <> + + + + {peak.count} + + + )} + + {/* Today dot */} + {points.length > 0 && ( + + )} + + {/* X-axis baseline */} + + + {/* Month labels */} + {monthLabels.map(({ x, label }, i) => ( + + {label} + + ))} + + {/* Invisible hit targets for per-day tooltip */} + {points.map((p) => ( + + {p.date}{p.count > 0 ? `: ${p.count} event${p.count !== 1 ? "s" : ""}` : ": no activity"} + + ))} + {/* ── Contributions summary ────────────────────────────────── */}

{totalEvents.toLocaleString()} - + contributions across all tracked repositories in the last 6 months

@@ -195,7 +258,6 @@ export function ForgejoHeatmap({ className="mx-auto flex max-w-lg items-start gap-3 rounded-lg px-4 py-2.5" style={{ background: "rgba(139,92,246,0.08)", border: "1px solid rgba(139,92,246,0.18)" }} > - {/* commit hash pill */} {lastPush.message}

-

+

{lastPush.author} pushed to{" "} {lastPush.branch} - {" · "} - {lastPush.repo} - {" · "} - {fmtRelative(lastPush.pushed_at)} + {" · "}{lastPush.repo}{" · "}{fmtRelative(lastPush.pushed_at)}

@@ -225,22 +284,18 @@ export function ForgejoHeatmap({ +{fmtLines(totalAdditions)} - - lines added - + lines added
-{fmtLines(totalDeletions)} - - lines removed - + lines removed
) : ( -

+

Line stats syncing with Forgejo — will appear on next refresh

)}