From e56f252da60ae9c4cdafee4efa38c601e721ebe6 Mon Sep 17 00:00:00 2001 From: null Date: Thu, 21 May 2026 22:47:24 -0500 Subject: [PATCH] feat(scripts): issues info --- backend/app/api/forgejo_issues.py | 85 ++++++++++++------- backend/app/api/forgejo_webhooks.py | 2 + backend/app/models/forgejo_issues.py | 3 + backend/app/schemas/forgejo_issues.py | 6 ++ backend/app/services/forgejo_issue_close.py | 12 ++- backend/app/services/forgejo_issue_sync.py | 47 ++++++---- ...5b6c7d8_add_forgejo_issue_payload_cache.py | 28 ++++++ backend/tests/test_forgejo_issue_close_api.py | 14 ++- backend/tests/test_forgejo_webhooks_api.py | 3 + 9 files changed, 146 insertions(+), 54 deletions(-) create mode 100644 backend/migrations/versions/d3f4a5b6c7d8_add_forgejo_issue_payload_cache.py diff --git a/backend/app/api/forgejo_issues.py b/backend/app/api/forgejo_issues.py index 8986445..0fd17d6 100644 --- a/backend/app/api/forgejo_issues.py +++ b/backend/app/api/forgejo_issues.py @@ -6,8 +6,8 @@ from typing import TYPE_CHECKING from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy import cast, String -from sqlmodel import select, func +from sqlalchemy import String, cast +from sqlmodel import func, select from app.api.deps import require_org_member from app.core.auth import AuthContext, get_auth_context @@ -15,9 +15,19 @@ from app.db import crud from app.db.session import get_session from app.models.board_repository_links import BoardRepositoryLink from app.models.forgejo_issues import ForgejoIssue -from app.schemas.forgejo_issues import ForgejoIssueListResponse, ForgejoIssueRead, CloseIssueResponse +from app.schemas.forgejo_issues import ( + CloseIssueResponse, + ForgejoIssueDetailRead, + ForgejoIssueListResponse, + ForgejoIssueRead, +) from app.services.activity_log import record_activity -from app.services.forgejo_issue_close import close_issue_by_id, CloseIssueNotFoundError, CloseIssueAccessError, CloseIssueRemoteError +from app.services.forgejo_issue_close import ( + CloseIssueAccessError, + CloseIssueNotFoundError, + CloseIssueRemoteError, + close_issue_by_id, +) from app.services.organizations import OrganizationContext, list_accessible_board_ids if TYPE_CHECKING: @@ -35,7 +45,9 @@ async def list_issues( session: AsyncSession = SESSION_DEP, ctx: OrganizationContext = ORG_MEMBER_DEP, repository_id: str | None = Query(None, description="Filter by repository ID"), - board_id: str | None = Query(None, description="Filter by board ID (returns issues from all repos linked to this board)"), + board_id: str | None = Query( + None, description="Filter by board ID (returns issues from all repos linked to this board)" + ), state: str | None = Query(None, description="Filter by state (open, closed)"), label: str | None = Query(None, description="Filter by label name"), assignee: str | None = Query(None, description="Filter by assignee login"), @@ -54,7 +66,9 @@ async def list_issues( try: board_uuid = UUID(board_id) except ValueError: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid board_id format") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid board_id format" + ) linked_repo_ids = ( await session.exec( select(BoardRepositoryLink.repository_id).where( @@ -71,7 +85,10 @@ async def list_issues( try: repo_uuid = UUID(repository_id) except ValueError: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid repository_id format") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail="Invalid repository_id format", + ) statement = statement.where(ForgejoIssue.repository_id == repo_uuid) if state: @@ -79,27 +96,27 @@ async def list_issues( if label: # Filter by label name — search within the JSON labels array cast to text - statement = statement.where( - cast(ForgejoIssue.labels, String).ilike(f"%{label}%") - ) + statement = statement.where(cast(ForgejoIssue.labels, String).ilike(f"%{label}%")) if assignee: # Filter by assignee login — search within the JSON assignees array cast to text - statement = statement.where( - cast(ForgejoIssue.assignees, String).ilike(f"%{assignee}%") - ) + statement = statement.where(cast(ForgejoIssue.assignees, String).ilike(f"%{assignee}%")) if search: statement = statement.where( - (ForgejoIssue.title.ilike(f"%{search}%")) | - (ForgejoIssue.body_preview.ilike(f"%{search}%")) | - (ForgejoIssue.body.ilike(f"%{search}%")) + (ForgejoIssue.title.ilike(f"%{search}%")) + | (ForgejoIssue.body_preview.ilike(f"%{search}%")) + | (ForgejoIssue.body.ilike(f"%{search}%")) ) # Count total - total_statement = select(func.count()).select_from(ForgejoIssue).where( - ForgejoIssue.organization_id == ctx.organization.id, - ForgejoIssue.is_pull_request.is_(False), + total_statement = ( + select(func.count()) + .select_from(ForgejoIssue) + .where( + ForgejoIssue.organization_id == ctx.organization.id, + ForgejoIssue.is_pull_request.is_(False), + ) ) if board_id: try: @@ -136,16 +153,18 @@ async def list_issues( ) if search: total_statement = total_statement.where( - (ForgejoIssue.title.ilike(f"%{search}%")) | - (ForgejoIssue.body_preview.ilike(f"%{search}%")) | - (ForgejoIssue.body.ilike(f"%{search}%")) + (ForgejoIssue.title.ilike(f"%{search}%")) + | (ForgejoIssue.body_preview.ilike(f"%{search}%")) + | (ForgejoIssue.body.ilike(f"%{search}%")) ) total_result = await session.exec(total_statement) total = total_result.one() # Pagination offset = (page - 1) * limit - statement = statement.offset(offset).limit(limit).order_by(ForgejoIssue.forgejo_issue_number.desc()) + statement = ( + statement.offset(offset).limit(limit).order_by(ForgejoIssue.forgejo_issue_number.desc()) + ) issues = (await session.exec(statement)).all() items = [ForgejoIssueRead.model_validate(issue) for issue in issues] @@ -158,17 +177,19 @@ async def list_issues( ) -@router.get("/{issue_id}", response_model=ForgejoIssueRead) +@router.get("/{issue_id}", response_model=ForgejoIssueDetailRead) async def get_issue( issue_id: str, session: AsyncSession = SESSION_DEP, ctx: OrganizationContext = ORG_MEMBER_DEP, -) -> ForgejoIssueRead: +) -> ForgejoIssueDetailRead: """Get one cached issue by ID.""" try: uuid = UUID(issue_id) except ValueError: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid issue_id format") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid issue_id format" + ) issue = await crud.get_by_id(session, ForgejoIssue, uuid) if issue is None: @@ -176,7 +197,7 @@ async def get_issue( if issue.organization_id != ctx.organization.id: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) - return ForgejoIssueRead.model_validate(issue) + return ForgejoIssueDetailRead.model_validate(issue) @router.post( @@ -224,14 +245,16 @@ async def close_issue( ctx: OrganizationContext = ORG_MEMBER_DEP, ) -> CloseIssueResponse: """Close a Forgejo issue as an authenticated user. - + The user must have write access to the board that the issue's repository is linked to. The issue must belong to a repository linked to that board. """ try: uuid = UUID(issue_id) except ValueError: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid issue_id format") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Invalid issue_id format" + ) issue = await crud.get_by_id(session, ForgejoIssue, uuid) if issue is None: @@ -256,9 +279,7 @@ async def close_issue( detail="Issue repository is not linked to any board", ) - allowed_board_ids = set( - await list_accessible_board_ids(session, member=ctx.member, write=True) - ) + allowed_board_ids = set(await list_accessible_board_ids(session, member=ctx.member, write=True)) authorized_board_id = next( (link.board_id for link in links if link.board_id in allowed_board_ids), None, diff --git a/backend/app/api/forgejo_webhooks.py b/backend/app/api/forgejo_webhooks.py index c6092d3..1d31cf9 100644 --- a/backend/app/api/forgejo_webhooks.py +++ b/backend/app/api/forgejo_webhooks.py @@ -323,6 +323,7 @@ async def _upsert_issue( labels=_labels(issue_data), assignees=_assignees(issue_data), milestone=milestone, + forgejo_payload=dict(issue_data), author=_author(issue_data), html_url=str(issue_data.get("html_url") or ""), forgejo_created_at=_parse_iso_datetime(issue_data.get("created_at")), @@ -341,6 +342,7 @@ async def _upsert_issue( existing.is_pull_request = False existing.labels = _labels(issue_data) existing.assignees = _assignees(issue_data) + existing.forgejo_payload = dict(issue_data) existing.author = _author(issue_data) existing.html_url = str(issue_data.get("html_url") or "") existing.forgejo_created_at = _parse_iso_datetime(issue_data.get("created_at")) diff --git a/backend/app/models/forgejo_issues.py b/backend/app/models/forgejo_issues.py index 4c479c0..391b2e1 100644 --- a/backend/app/models/forgejo_issues.py +++ b/backend/app/models/forgejo_issues.py @@ -34,6 +34,9 @@ class ForgejoIssue(SQLModel, table=True): labels: list[dict[str, object]] = Field(default_factory=list, sa_column=Column(JSON)) assignees: list[dict[str, object]] = Field(default_factory=list, sa_column=Column(JSON)) milestone: dict[str, object] | None = Field(default=None, sa_column=Column(JSON, nullable=True)) + forgejo_payload: dict[str, object] | None = Field( + default=None, sa_column=Column(JSON, nullable=True) + ) author: str html_url: str diff --git a/backend/app/schemas/forgejo_issues.py b/backend/app/schemas/forgejo_issues.py index d295ff4..37576eb 100644 --- a/backend/app/schemas/forgejo_issues.py +++ b/backend/app/schemas/forgejo_issues.py @@ -42,6 +42,12 @@ class ForgejoIssueRead(ForgejoIssueBase): updated_at: datetime +class ForgejoIssueDetailRead(ForgejoIssueRead): + """Issue detail payload including full cached Forgejo source payload.""" + + forgejo_payload: dict[str, Any] | None = None + + class ForgejoIssueListResponse(SQLModel): """Paginated list response for issues.""" diff --git a/backend/app/services/forgejo_issue_close.py b/backend/app/services/forgejo_issue_close.py index 07c4043..09120df 100644 --- a/backend/app/services/forgejo_issue_close.py +++ b/backend/app/services/forgejo_issue_close.py @@ -90,7 +90,13 @@ async def close_cached_issue( issue.state = "closed" issue.forgejo_closed_at = utcnow() issue.last_synced_at = utcnow() - + if isinstance(issue.forgejo_payload, dict): + payload = dict(issue.forgejo_payload) + payload["state"] = "closed" + payload["closed_at"] = issue.forgejo_closed_at.isoformat() + payload["updated_at"] = issue.last_synced_at.isoformat() + issue.forgejo_payload = payload + # Update the issue in the session session.add(issue) await session.flush() @@ -114,7 +120,9 @@ async def close_cached_issue( "repository_id": str(repository.id), "repository_full_name": f"{repository.owner}/{repository.repo}", "state": "closed", - "forgejo_closed_at": issue.forgejo_closed_at.isoformat() if issue.forgejo_closed_at else None, + "forgejo_closed_at": ( + issue.forgejo_closed_at.isoformat() if issue.forgejo_closed_at else None + ), "last_synced_at": issue.last_synced_at.isoformat(), } diff --git a/backend/app/services/forgejo_issue_sync.py b/backend/app/services/forgejo_issue_sync.py index b23ecc3..64f3dc4 100644 --- a/backend/app/services/forgejo_issue_sync.py +++ b/backend/app/services/forgejo_issue_sync.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import datetime -from typing import Any from uuid import UUID from sqlmodel import select @@ -14,7 +13,7 @@ from app.db import crud from app.models.forgejo_connections import ForgejoConnection from app.models.forgejo_issues import ForgejoIssue from app.models.forgejo_repositories import ForgejoRepository -from app.services.forgejo_client import ForgejoAPIClient, get_forgejo_client +from app.services.forgejo_client import get_forgejo_client logger = get_logger(__name__) @@ -63,7 +62,11 @@ class IssueSyncService: ) # Forgejo returns issues as a JSON array, not wrapped in "items" - issues = response if isinstance(response, list) else response.get("items", response.get("data", [])) + issues = ( + response + if isinstance(response, list) + else response.get("items", response.get("data", [])) + ) if not isinstance(issues, list) or len(issues) == 0: break @@ -77,22 +80,26 @@ class IssueSyncService: # Parse labels labels_data = [] - for label in (issue_data.get("labels") or []): - labels_data.append({ - "id": label.get("id"), - "name": label.get("name", ""), - "color": label.get("color", ""), - "description": label.get("description", ""), - }) + for label in issue_data.get("labels") or []: + labels_data.append( + { + "id": label.get("id"), + "name": label.get("name", ""), + "color": label.get("color", ""), + "description": label.get("description", ""), + } + ) # Parse assignees assignees_data = [] - for assignee in (issue_data.get("assignees") or []): - assignees_data.append({ - "login": assignee.get("login", ""), - "id": assignee.get("id", 0), - "avatar_url": assignee.get("avatar_url", ""), - }) + for assignee in issue_data.get("assignees") or []: + assignees_data.append( + { + "login": assignee.get("login", ""), + "id": assignee.get("id", 0), + "avatar_url": assignee.get("avatar_url", ""), + } + ) # Parse milestone milestone_data = None @@ -133,6 +140,7 @@ class IssueSyncService: labels=labels_data, assignees=assignees_data, milestone=milestone_data, + forgejo_payload=dict(issue_data), author=issue_data.get("user", {}).get("login", ""), html_url=issue_data.get("html_url", ""), forgejo_created_at=created_at, @@ -150,6 +158,7 @@ class IssueSyncService: existing.labels = labels_data existing.assignees = assignees_data existing.milestone = milestone_data + existing.forgejo_payload = dict(issue_data) existing.author = issue_data.get("user", {}).get("login", "") existing.html_url = issue_data.get("html_url", "") existing.forgejo_created_at = created_at @@ -225,7 +234,9 @@ class IssueSyncService: "total": created + updated_count, } - async def _find_issue(self, repository_id: UUID, forgejo_issue_number: int) -> ForgejoIssue | None: + async def _find_issue( + self, repository_id: UUID, forgejo_issue_number: int + ) -> ForgejoIssue | None: """Find an existing cached issue by repository and number.""" statement = select(ForgejoIssue).where( ForgejoIssue.repository_id == repository_id, @@ -243,4 +254,4 @@ class IssueSyncService: parsed = datetime.fromisoformat(cleaned) return parsed.replace(tzinfo=None) except (ValueError, AttributeError): - return None \ No newline at end of file + return None diff --git a/backend/migrations/versions/d3f4a5b6c7d8_add_forgejo_issue_payload_cache.py b/backend/migrations/versions/d3f4a5b6c7d8_add_forgejo_issue_payload_cache.py new file mode 100644 index 0000000..0485148 --- /dev/null +++ b/backend/migrations/versions/d3f4a5b6c7d8_add_forgejo_issue_payload_cache.py @@ -0,0 +1,28 @@ +"""add cached raw Forgejo payload on issues + +Revision ID: d3f4a5b6c7d8 +Revises: a7aa29cf8a20 +Create Date: 2026-05-22 00:00:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "d3f4a5b6c7d8" +down_revision = "a7aa29cf8a20" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "forgejo_issues", + sa.Column("forgejo_payload", sa.JSON(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("forgejo_issues", "forgejo_payload") diff --git a/backend/tests/test_forgejo_issue_close_api.py b/backend/tests/test_forgejo_issue_close_api.py index 0ad3d7b..c2eedd5 100644 --- a/backend/tests/test_forgejo_issue_close_api.py +++ b/backend/tests/test_forgejo_issue_close_api.py @@ -17,8 +17,8 @@ from sqlmodel.ext.asyncio.session import AsyncSession from app import models as _models from app.api import deps as deps_api from app.api.agent_forgejo import router as agent_forgejo_router -from app.api.forgejo_issues import router as forgejo_issues_router from app.api.deps import require_org_member +from app.api.forgejo_issues import router as forgejo_issues_router from app.core.agent_auth import AgentAuthContext, get_agent_auth_context from app.core.auth import AuthContext, get_auth_context from app.db.session import get_session @@ -66,28 +66,34 @@ def _build_test_app( app.dependency_overrides[get_session] = _override_get_session if org_ctx is not None: + async def _override_require_org_member() -> OrganizationContext: return org_ctx app.dependency_overrides[require_org_member] = _override_require_org_member if auth_ctx is not None: + async def _override_get_auth_context() -> AuthContext: return auth_ctx app.dependency_overrides[get_auth_context] = _override_get_auth_context if agent_ctx is not None: + async def _override_get_agent_auth_context() -> AgentAuthContext: return agent_ctx app.dependency_overrides[get_agent_auth_context] = _override_get_agent_auth_context if board is not None: + async def _override_get_board_for_actor_read() -> Board: return board - app.dependency_overrides[deps_api.get_board_for_actor_read] = _override_get_board_for_actor_read + app.dependency_overrides[deps_api.get_board_for_actor_read] = ( + _override_get_board_for_actor_read + ) return app @@ -155,6 +161,7 @@ async def _seed(session: AsyncSession) -> SimpleNamespace: is_pull_request=False, labels=[], assignees=[], + forgejo_payload={"number": 42, "state": "open"}, author="kaspa", html_url="https://forgejo.example.local/openclaw/pipeline/issues/42", forgejo_created_at=datetime(2026, 5, 19, 12, 0, 0), @@ -267,6 +274,9 @@ async def test_human_writer_can_close_linked_issue_and_records_activity( assert stored_issue.state == "closed" assert stored_issue.forgejo_closed_at is not None assert stored_issue.last_synced_at is not None + assert stored_issue.forgejo_payload is not None + assert stored_issue.forgejo_payload.get("state") == "closed" + assert stored_issue.forgejo_payload.get("closed_at") events = (await session.exec(select(ActivityEvent))).all() assert len(events) == 1 diff --git a/backend/tests/test_forgejo_webhooks_api.py b/backend/tests/test_forgejo_webhooks_api.py index 1a36650..b0a170c 100644 --- a/backend/tests/test_forgejo_webhooks_api.py +++ b/backend/tests/test_forgejo_webhooks_api.py @@ -243,6 +243,9 @@ async def test_forgejo_webhook_closes_issue_and_records_board_activity() -> None assert stored_issue.assignees == [ {"login": "codex", "id": 7, "avatar_url": "https://avatar.test/c"} ] + assert isinstance(stored_issue.forgejo_payload, dict) + assert stored_issue.forgejo_payload.get("number") == 42 + assert stored_issue.forgejo_payload.get("title") == "Fix webhook cache updates" assert stored_issue.author == "kaspa" assert stored_issue.forgejo_closed_at == datetime(2026, 5, 19, 12, 45, 0)