null
/
Pipeline
1
0
Fork 0
Code Issues 15 Pull Requests Packages Projects Releases Wiki Activity

heatmap commits

This commit is contained in:
null 2026-05-24 20:13:04 -05:00
parent 086fc8fc49
commit 1fc1df5aeb
2 changed files with 75 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
backend/app/api/forgejo_metrics.py
View File

@ -30,11 +30,12 @@ if TYPE_CHECKING:
# ---------------------------------------------------------------------------
# Line-stats background cache
# ---------------------------------------------------------------------------
# Key: org_id string → (fetched_at, total_additions, total_deletions, has_data)
# Key: org_id string → (fetched_at, total_additions, total_deletions, has_data, day_counts)
# day_counts maps "YYYY-MM-DD" → commit count for the heatmap grid.
# Populated by a fire-and-forget asyncio task so the heatmap endpoint never
# blocks waiting for Forgejo's 202 "still computing" response.
# ---------------------------------------------------------------------------
_line_stats_cache: dict[str, tuple[float, int, int, bool]] = {}
_line_stats_cache: dict[str, tuple[float, int, int, bool, dict[str, int]]] = {}
_line_stats_fetching: set[str] = set()
_LINE_STATS_TTL_HIT = 300 # 5 min — re-fetch cadence once real data is cached
_LINE_STATS_TTL_MISS = 30 # 30 s — retry cadence while Forgejo is still computing
@ -47,19 +48,23 @@ async def _bg_fetch_line_stats(
) -> None:
"""Background task: sum commit line stats across all tracked repos and cache."""
async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int]:
async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int, dict[str, int]]:
try:
async with ForgejoAPIClient(base_url=base_url, token=token) as client:
return await client.get_commit_line_stats_since(owner, repo, since_iso)
except Exception:
return 0, 0
return 0, 0, {}
try:
results = await asyncio.gather(*[_one(o, r, bu, tok) for o, r, bu, tok in repos])
total_adds = sum(a for a, _ in results)
total_dels = sum(d for _, d in results)
total_adds = sum(a for a, _, _ in results)
total_dels = sum(d for _, d, _ in results)
merged_days: dict[str, int] = {}
for _, _, day_counts in results:
for day, cnt in day_counts.items():
merged_days[day] = merged_days.get(day, 0) + cnt
# Always mark has_data=True — the commits endpoint is reliable
_line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, True)
_line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, True, merged_days)
finally:
_line_stats_fetching.discard(cache_key)
@ -400,51 +405,8 @@ async def get_forgejo_heatmap(
return HeatmapResponse(days=[], max_count=0)
repo_ids = [repo.id for repo, _ in repos_with_conns]
counts: dict[str, int] = {}
# Issues created per day
created_rows = (
await session.exec(
select(
sa_cast(ForgejoIssue.forgejo_created_at, SADate).label("day"),
func.count().label("cnt"),
).where(
ForgejoIssue.repository_id.in_(repo_ids),
ForgejoIssue.is_pull_request.is_(False),
ForgejoIssue.forgejo_created_at.is_not(None),
ForgejoIssue.forgejo_created_at >= since,
).group_by(sa_cast(ForgejoIssue.forgejo_created_at, SADate))
)
).all()
for day, cnt in created_rows:
if day:
key = str(day)
counts[key] = counts.get(key, 0) + int(cnt)
# Issues closed per day
closed_rows = (
await session.exec(
select(
sa_cast(ForgejoIssue.forgejo_closed_at, SADate).label("day"),
func.count().label("cnt"),
).where(
ForgejoIssue.repository_id.in_(repo_ids),
ForgejoIssue.is_pull_request.is_(False),
ForgejoIssue.forgejo_closed_at.is_not(None),
ForgejoIssue.forgejo_closed_at >= since,
).group_by(sa_cast(ForgejoIssue.forgejo_closed_at, SADate))
)
).all()
for day, cnt in closed_rows:
if day:
key = str(day)
counts[key] = counts.get(key, 0) + int(cnt)
days_list = [HeatmapDay(date=k, count=v) for k, v in sorted(counts.items())]
max_count = max((d.count for d in days_list), default=0)
# Line stats — served from cache; background task refreshes when stale.
# Extract plain values NOW while the DB session is still open.
# Line stats — served from background cache; fire refresh if stale.
since_iso = line_stats_since.strftime("%Y-%m-%dT%H:%M:%SZ")
cache_key = str(ctx.organization.id)
cached = _line_stats_cache.get(cache_key)
@ -472,10 +434,58 @@ async def get_forgejo_heatmap(
)
if cached is not None:
_, total_additions, total_deletions, has_line_stats = cached
_, total_additions, total_deletions, has_line_stats, commit_day_counts = cached
else:
total_additions = total_deletions = 0
has_line_stats = False
commit_day_counts = {}
# Heatmap grid: use commit-per-day counts when available; fall back to
# issue-event counts (from DB) while the background fetch is in progress.
if has_line_stats and commit_day_counts:
days_list = [HeatmapDay(date=k, count=v) for k, v in sorted(commit_day_counts.items())]
max_count = max((d.count for d in days_list), default=0)
else:
counts: dict[str, int] = {}
created_rows = (
await session.exec(
select(
sa_cast(ForgejoIssue.forgejo_created_at, SADate).label("day"),
func.count().label("cnt"),
).where(
ForgejoIssue.repository_id.in_(repo_ids),
ForgejoIssue.is_pull_request.is_(False),
ForgejoIssue.forgejo_created_at.is_not(None),
ForgejoIssue.forgejo_created_at >= since,
).group_by(sa_cast(ForgejoIssue.forgejo_created_at, SADate))
)
).all()
for day, cnt in created_rows:
if day:
key = str(day)
counts[key] = counts.get(key, 0) + int(cnt)
closed_rows = (
await session.exec(
select(
sa_cast(ForgejoIssue.forgejo_closed_at, SADate).label("day"),
func.count().label("cnt"),
).where(
ForgejoIssue.repository_id.in_(repo_ids),
ForgejoIssue.is_pull_request.is_(False),
ForgejoIssue.forgejo_closed_at.is_not(None),
ForgejoIssue.forgejo_closed_at >= since,
).group_by(sa_cast(ForgejoIssue.forgejo_closed_at, SADate))
)
).all()
for day, cnt in closed_rows:
if day:
key = str(day)
counts[key] = counts.get(key, 0) + int(cnt)
days_list = [HeatmapDay(date=k, count=v) for k, v in sorted(counts.items())]
max_count = max((d.count for d in days_list), default=0)
return HeatmapResponse(
days=days_list,

17
backend/app/services/forgejo_client.py
View File

@ -374,15 +374,17 @@ class ForgejoAPIClient:
async def get_commit_line_stats_since(
self, owner: str, repo: str, since_iso: str
) -> tuple[int, int]:
"""Sum additions and deletions across all commits since ``since_iso``.
) -> tuple[int, int, dict[str, int]]:
"""Sum additions/deletions and count commits per day since ``since_iso``.
Uses ``GET /repos/{owner}/{repo}/commits?since=…&stat=true`` which
returns per-commit ``stats`` objects and is available on all Forgejo/
Gitea versions. Returns ``(total_additions, total_deletions)``.
Gitea versions. Returns ``(total_additions, total_deletions, day_counts)``
where ``day_counts`` maps "YYYY-MM-DD" → commit count.
"""
client = await self._get_client()
total_adds = total_dels = 0
day_counts: dict[str, int] = {}
page = 1
# Cap at 5 pages (250 commits) — prevents unbounded pagination on
# very active repos where each page requires Forgejo to compute
@ -403,10 +405,17 @@ class ForgejoAPIClient:
s = commit.get("stats") or {}
total_adds += int(s.get("additions") or 0)
total_dels += int(s.get("deletions") or 0)
# Extract commit date for per-day activity counting
commit_obj = commit.get("commit") or {}
author_obj = commit_obj.get("author") or {}
date_str: str = author_obj.get("date") or commit.get("created") or ""
if date_str and len(date_str) >= 10:
day = date_str[:10]
day_counts[day] = day_counts.get(day, 0) + 1
if len(commits) < 50:
break # last page
page += 1
return total_adds, total_dels
return total_adds, total_dels, day_counts
def get_forgejo_client(