From 7919c29bbd854c00b141896f699abe1ecfd0e9df Mon Sep 17 00:00:00 2001 From: null Date: Sun, 24 May 2026 20:24:43 -0500 Subject: [PATCH] New table: forgejo_commit_activity --- backend/app/api/forgejo_metrics.py | 157 ++++++++++++++++-- backend/app/models/forgejo_commit_activity.py | 34 ++++ backend/app/schemas/metrics.py | 7 + ...1c2d3e4f5a6_add_forgejo_commit_activity.py | 62 +++++++ 4 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 backend/app/models/forgejo_commit_activity.py create mode 100644 backend/migrations/versions/b1c2d3e4f5a6_add_forgejo_commit_activity.py diff --git a/backend/app/api/forgejo_metrics.py b/backend/app/api/forgejo_metrics.py index 1b72621..ece5de0 100644 --- a/backend/app/api/forgejo_metrics.py +++ b/backend/app/api/forgejo_metrics.py @@ -16,12 +16,13 @@ from sqlmodel import func, select from app.api.deps import ORG_MEMBER_DEP, OrganizationContext from app.core.time import utcnow -from app.db.session import get_session +from app.db.session import async_session_maker, get_session from app.models.board_repository_links import BoardRepositoryLink +from app.models.forgejo_commit_activity import ForgejoCommitDay from app.models.forgejo_connections import ForgejoConnection from app.models.forgejo_issues import ForgejoIssue from app.models.forgejo_repositories import ForgejoRepository -from app.schemas.metrics import HeatmapDay, HeatmapResponse, LastPushRead, MetricsResponse +from app.schemas.metrics import CommitActivityDay, HeatmapDay, HeatmapResponse, LastPushRead, MetricsResponse from app.services.forgejo_client import ForgejoAPIClient if TYPE_CHECKING: @@ -43,10 +44,11 @@ _LINE_STATS_TTL_MISS = 30 # 30 s — retry cadence while Forgejo is still com async def _bg_fetch_line_stats( cache_key: str, - repos: list[tuple[str, str, str, str | None]], # (owner, repo, base_url, token) - since_iso: str, # ISO-8601 date string for the commits?since= filter + # (owner, repo, base_url, token, repository_id, organization_id) + repos: list[tuple[str, str, str, str | None, UUID, UUID]], + since_iso: str, ) -> None: - """Background task: sum commit line stats across all tracked repos and cache.""" + """Background task: sum commit line stats, persist per-repo day counts to DB.""" async def _one(owner: str, repo: str, base_url: str, token: str | None) -> tuple[int, int, dict[str, int]]: try: @@ -56,14 +58,52 @@ async def _bg_fetch_line_stats( return 0, 0, {} try: - results = await asyncio.gather(*[_one(o, r, bu, tok) for o, r, bu, tok in repos]) + results = await asyncio.gather(*[_one(o, r, bu, tok) for o, r, bu, tok, _, _ in repos]) total_adds = sum(a for a, _, _ in results) total_dels = sum(d for _, d, _ in results) merged_days: dict[str, int] = {} + + # Persist per-repo per-day counts to DB (upsert). + try: + from datetime import date as _date + async with async_session_maker() as session: + for (_, _, _, _, repo_id, org_id), (_, _, day_counts) in zip(repos, results): + for day_str, cnt in day_counts.items(): + try: + day_obj = _date.fromisoformat(day_str) + except ValueError: + continue + existing = ( + await session.exec( + select(ForgejoCommitDay).where( + ForgejoCommitDay.repository_id == repo_id, + ForgejoCommitDay.date == day_obj, + ) + ) + ).first() + if existing is None: + session.add( + ForgejoCommitDay( + organization_id=org_id, + repository_id=repo_id, + date=day_obj, + commit_count=cnt, + ) + ) + elif existing.commit_count != cnt: + existing.commit_count = cnt + existing.updated_at = utcnow() + await session.commit() + except Exception as db_exc: + from app.core.logging import get_logger as _get_logger + _get_logger(__name__).warning( + "commit_activity_db_write_failed", extra={"error": str(db_exc)} + ) + for _, _, day_counts in results: for day, cnt in day_counts.items(): merged_days[day] = merged_days.get(day, 0) + cnt - # Always mark has_data=True — the commits endpoint is reliable + _line_stats_cache[cache_key] = (_time.monotonic(), total_adds, total_dels, True, merged_days) finally: _line_stats_fetching.discard(cache_key) @@ -419,13 +459,13 @@ async def get_forgejo_heatmap( # Normalise base_url the same way get_forgejo_client() does, eagerly, # so the background task never touches a potentially-closed session. import re as _re - repo_tuples: list[tuple[str, str, str, str | None]] = [] + repo_tuples: list[tuple[str, str, str, str | None, UUID, UUID]] = [] for repo, conn in repos_with_conns: bu = (conn.base_url or "").rstrip("/") if "/api/v1" in bu: m = _re.match(r"(https?://[^/]+)", bu) bu = m.group(1).rstrip("/") if m else bu - repo_tuples.append((repo.owner, repo.repo, bu, getattr(conn, "token", None))) + repo_tuples.append((repo.owner, repo.repo, bu, getattr(conn, "token", None), repo.id, repo.organization_id)) _line_stats_fetching.add(cache_key) asyncio.create_task( @@ -440,12 +480,30 @@ async def get_forgejo_heatmap( has_line_stats = False commit_day_counts = {} - # Heatmap grid: use commit-per-day counts when available; fall back to - # issue-event counts (from DB) while the background fetch is in progress. - if has_line_stats and commit_day_counts: + # Heatmap grid: prefer commit-per-day counts from DB (persistent, no restarts); + # fall back to in-memory cache, then to issue-event counts while the first + # background fetch is in progress. + commit_db_rows = ( + await session.exec( + select(ForgejoCommitDay.date, func.sum(ForgejoCommitDay.commit_count).label("cnt")) + .where( + ForgejoCommitDay.repository_id.in_(repo_ids), + ForgejoCommitDay.date >= line_stats_since.date(), + ) + .group_by(ForgejoCommitDay.date) + .order_by(ForgejoCommitDay.date) + ) + ).all() + + if commit_db_rows: + days_list = [HeatmapDay(date=str(day), count=int(cnt)) for day, cnt in commit_db_rows] + max_count = max((d.count for d in days_list), default=0) + elif has_line_stats and commit_day_counts: + # In-memory cache hit but DB write hasn't landed yet days_list = [HeatmapDay(date=k, count=v) for k, v in sorted(commit_day_counts.items())] max_count = max((d.count for d in days_list), default=0) else: + # First-ever load — fall back to issue-event counts from DB while background fetch runs counts: dict[str, int] = {} created_rows = ( @@ -545,6 +603,81 @@ async def get_last_push( return cached[1] if cached is not None else None +@router.get( + "/commit-activity", + response_model=list[CommitActivityDay], + summary="Per-day commit counts from persistent DB store", + description=( + "Returns commit counts per calendar day from the forgejo_commit_activity table. " + "Populated by the background line-stats fetch; accurate across restarts. " + "Scope by organization_id or repository_id; defaults to the caller's full org." + ), +) +async def get_commit_activity( + organization_id: UUID | None = Query(None, description="Filter by organisation ID"), + repository_id: UUID | None = Query(None, description="Filter by a single repository ID"), + days: int = Query(default=90, ge=1, le=365, description="How many days back to return"), + session: AsyncSession = SESSION_DEP, + ctx: OrganizationContext = ORG_MEMBER_DEP, +) -> list[CommitActivityDay]: + """Return aggregated commit-per-day counts from the DB.""" + if organization_id and organization_id != ctx.organization.id: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN) + + since = (utcnow() - timedelta(days=days)).date() + + if repository_id: + # Verify repo belongs to this org + repo = ( + await session.exec( + select(ForgejoRepository).where( + ForgejoRepository.id == repository_id, + ForgejoRepository.organization_id == ctx.organization.id, + ) + ) + ).first() + if repo is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + rows = ( + await session.exec( + select(ForgejoCommitDay.date, ForgejoCommitDay.commit_count) + .where( + ForgejoCommitDay.repository_id == repository_id, + ForgejoCommitDay.date >= since, + ) + .order_by(ForgejoCommitDay.date) + ) + ).all() + else: + # Org-wide: aggregate across all repos + repo_ids_rows = ( + await session.exec( + select(ForgejoRepository.id).where( + ForgejoRepository.organization_id == ctx.organization.id + ) + ) + ).all() + repo_ids = list(repo_ids_rows) + if not repo_ids: + return [] + rows = ( + await session.exec( + select( + ForgejoCommitDay.date, + func.sum(ForgejoCommitDay.commit_count).label("commit_count"), + ) + .where( + ForgejoCommitDay.repository_id.in_(repo_ids), + ForgejoCommitDay.date >= since, + ) + .group_by(ForgejoCommitDay.date) + .order_by(ForgejoCommitDay.date) + ) + ).all() + + return [CommitActivityDay(date=str(day), commit_count=int(cnt)) for day, cnt in rows] + + def _zeroed_metrics() -> MetricsResponse: """Return zeroed metrics for empty scopes.""" return MetricsResponse( diff --git a/backend/app/models/forgejo_commit_activity.py b/backend/app/models/forgejo_commit_activity.py new file mode 100644 index 0000000..029c8a7 --- /dev/null +++ b/backend/app/models/forgejo_commit_activity.py @@ -0,0 +1,34 @@ +"""Per-repository per-day commit activity for heatmap and trend queries.""" + +from __future__ import annotations + +from datetime import date, datetime +from uuid import UUID, uuid4 + +from sqlmodel import Field, Index, SQLModel + +from app.core.time import utcnow + + +class ForgejoCommitDay(SQLModel, table=True): + """One row per repository per calendar day recording commit count.""" + + __tablename__ = "forgejo_commit_activity" # pyright: ignore[reportAssignmentType] + + id: UUID = Field(default_factory=uuid4, primary_key=True) + organization_id: UUID = Field(foreign_key="organizations.id", index=True) + repository_id: UUID = Field(foreign_key="forgejo_repositories.id", index=True) + date: date = Field(index=True) + commit_count: int = Field(default=0) + + created_at: datetime = Field(default_factory=utcnow) + updated_at: datetime = Field(default_factory=utcnow) + + __table_args__ = ( + Index( + "ix_forgejo_commit_activity_repo_date", + "repository_id", + "date", + unique=True, + ), + ) diff --git a/backend/app/schemas/metrics.py b/backend/app/schemas/metrics.py index f0bd335..8cddabd 100644 --- a/backend/app/schemas/metrics.py +++ b/backend/app/schemas/metrics.py @@ -149,6 +149,13 @@ class HeatmapResponse(SQLModel): has_line_stats: bool = False # False when Forgejo is still computing stats (HTTP 202) +class CommitActivityDay(SQLModel): + """Per-day commit count for a single repository or aggregated across an org.""" + + date: str # "YYYY-MM-DD" + commit_count: int + + class MetricsResponse(SQLModel): """Generic metrics response wrapper.""" diff --git a/backend/migrations/versions/b1c2d3e4f5a6_add_forgejo_commit_activity.py b/backend/migrations/versions/b1c2d3e4f5a6_add_forgejo_commit_activity.py new file mode 100644 index 0000000..635c791 --- /dev/null +++ b/backend/migrations/versions/b1c2d3e4f5a6_add_forgejo_commit_activity.py @@ -0,0 +1,62 @@ +"""Add forgejo_commit_activity table for per-repository per-day commit counts. + +Revision ID: b1c2d3e4f5a6 +Revises: a1b2c3d4e5f9, a3c5e7f9b1d2 +Create Date: 2026-05-24 00:00:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "b1c2d3e4f5a6" +down_revision = ("a1b2c3d4e5f9", "a3c5e7f9b1d2") +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "forgejo_commit_activity", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("organization_id", sa.Uuid(), nullable=False), + sa.Column("repository_id", sa.Uuid(), nullable=False), + sa.Column("date", sa.Date(), nullable=False), + sa.Column("commit_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["organization_id"], ["organizations.id"]), + sa.ForeignKeyConstraint(["repository_id"], ["forgejo_repositories.id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_forgejo_commit_activity_organization_id", + "forgejo_commit_activity", + ["organization_id"], + ) + op.create_index( + "ix_forgejo_commit_activity_repository_id", + "forgejo_commit_activity", + ["repository_id"], + ) + op.create_index( + "ix_forgejo_commit_activity_date", + "forgejo_commit_activity", + ["date"], + ) + op.create_index( + "ix_forgejo_commit_activity_repo_date", + "forgejo_commit_activity", + ["repository_id", "date"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("ix_forgejo_commit_activity_repo_date", "forgejo_commit_activity") + op.drop_index("ix_forgejo_commit_activity_date", "forgejo_commit_activity") + op.drop_index("ix_forgejo_commit_activity_repository_id", "forgejo_commit_activity") + op.drop_index("ix_forgejo_commit_activity_organization_id", "forgejo_commit_activity") + op.drop_table("forgejo_commit_activity")