From d6d094a67d1f55950c40d51a62219c907f25d185 Mon Sep 17 00:00:00 2001 From: null Date: Tue, 19 May 2026 21:18:18 -0500 Subject: [PATCH] Feat: Settings git --- .gitignore | 2 + FUTURE.md | 27 +- backend/.env.example | 4 + backend/README.md | 9 + backend/app/api/forgejo_connections.py | 50 ++ backend/app/core/config.py | 5 + backend/app/main.py | 2 + backend/app/services/forgejo_client.py | 24 + backend/app/services/forgejo_sync_queue.py | 147 ++++ backend/app/services/queue_worker.py | 16 + backend/tests/test_forgejo_client.py | 20 +- backend/tests/test_forgejo_sync_queue.py | 230 ++++++ compose.yml | 3 + docs/reference/configuration.md | 15 + .../repositories/[repositoryId]/edit/page.tsx | 1 + .../src/app/settings/git-projects/page.tsx | 688 ++++++++++++++++++ frontend/src/app/settings/page.tsx | 33 +- .../components/git/ForgejoRepositoryForm.tsx | 314 ++++++-- .../components/organisms/DashboardSidebar.tsx | 7 + frontend/src/lib/api-forgejo.ts | 21 + 20 files changed, 1541 insertions(+), 77 deletions(-) create mode 100644 backend/app/services/forgejo_sync_queue.py create mode 100644 backend/tests/test_forgejo_sync_queue.py create mode 100644 frontend/src/app/settings/git-projects/page.tsx diff --git a/.gitignore b/.gitignore index 8434635..0bb5bb6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ backend/coverage.* backend/.coverage frontend/coverage backend/app/services/openclaw/.device-keys +FUTURE.md +FUTURE.md diff --git a/FUTURE.md b/FUTURE.md index 51ec6c0..27b8513 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -1,26 +1,27 @@ # Pipeline — Up Next ## Phase 1: Backend — Forgejo API Integration -- [ ] Forgejo API client service (`backend/app/services/forgejo/`) -- [ ] DB models: `ForgejoRepo`, `ForgejoIssue`, `ForgejoLabel` -- [ ] Alembic migration for new models -- [ ] API endpoints: `/api/v1/forgejo/repos`, `/api/v1/forgejo/issues` -- [ ] Config: `FORGEJO_URL`, `FORGEJO_TOKEN`, `FORGEJO_REPOS` env vars -- [ ] On-demand sync endpoint: `POST /api/v1/forgejo/sync` -- [ ] Scheduled sync (background task or cron) +- [x] Forgejo API client service (`backend/app/services/forgejo_client.py`) +- [x] DB models: `ForgejoConnection`, `ForgejoRepository`, `ForgejoIssue` (labels cached on issues) +- [x] Alembic migration for new models +- [x] API endpoints: `/api/v1/forgejo/repositories`, `/api/v1/forgejo/issues` +- [x] Config via Forgejo connection/repository CRUD +- [x] On-demand repository sync endpoint: `POST /api/v1/forgejo/repositories/{repository_id}/sync` +- [x] Scheduled sync (background task or cron) ## Phase 2: Frontend — Issues Panel -- [ ] New "Issues" sidebar link and route -- [ ] Issue list page with repo/label/priority filters +- [x] New "Issues" sidebar link and route (`/git-projects/issues`) +- [x] Issue list page with repo/state/search filters +- [ ] Label/priority filters - [ ] Issue detail page - [ ] "Create Task from Issue" button ## Phase 3: Task Integration - [ ] Link tasks to Forgejo issues (foreign key on task) - [ ] Task detail view shows linked issue info -- [ ] Dashboard issue count widgets +- [x] Dashboard issue count widgets ## Phase 4: Polish -- [ ] Forgejo webhook for real-time updates -- [ ] Issue status sync (open/closed) -- [ ] Auto-archive closed issues \ No newline at end of file +- [x] Forgejo webhook for cached issue updates +- [x] Issue status sync (open/closed) +- [ ] Auto-archive closed issues diff --git a/backend/.env.example b/backend/.env.example index 422ad9c..355afc4 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -31,4 +31,8 @@ RQ_REDIS_URL=redis://localhost:6379/0 RQ_QUEUE_NAME=default RQ_DISPATCH_THROTTLE_SECONDS=15.0 RQ_DISPATCH_MAX_RETRIES=3 +# Forgejo issue cache scheduled sync +FORGEJO_SYNC_ENABLED=true +FORGEJO_SYNC_INTERVAL_SECONDS=3600 +FORGEJO_SYNC_STARTUP_DELAY_SECONDS=60 GATEWAY_MIN_VERSION=2026.02.9 diff --git a/backend/README.md b/backend/README.md index 099153b..267b8cb 100644 --- a/backend/README.md +++ b/backend/README.md @@ -65,6 +65,15 @@ A starter file exists at `backend/.env.example`. - If `true`: on startup, the backend attempts to run Alembic migrations (`alembic upgrade head`). - If there are **no** Alembic revision files yet, it falls back to `SQLModel.metadata.create_all`. +### Forgejo scheduled sync + +- `FORGEJO_SYNC_ENABLED` (default: `true`) + - When enabled, the API seeds a periodic Forgejo issue-cache sync task into the RQ queue on startup. +- `FORGEJO_SYNC_INTERVAL_SECONDS` (default: `3600`) + - How often the worker syncs all active Forgejo repositories. +- `FORGEJO_SYNC_STARTUP_DELAY_SECONDS` (default: `60`) + - Initial delay before the first scheduled sync after API startup. + ### Security headers Security response headers added to every API response. Set any variable to blank to disable the corresponding header. diff --git a/backend/app/api/forgejo_connections.py b/backend/app/api/forgejo_connections.py index b43345c..5dd131d 100644 --- a/backend/app/api/forgejo_connections.py +++ b/backend/app/api/forgejo_connections.py @@ -172,6 +172,56 @@ async def delete_connection( return OkResponse() +@router.get( + "/{connection_id}/repos", + summary="List Available Repositories", + description="Return repositories the connection token can access on the Forgejo instance.", +) +async def list_connection_repos( + connection_id: UUID, + session: AsyncSession = SESSION_DEP, + ctx: OrganizationContext = ORG_MEMBER_DEP, +) -> list[dict[str, object]]: + """List repositories accessible via this connection's token.""" + connection = await crud.get_by_id(session, ForgejoConnection, connection_id) + if connection is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + if connection.organization_id != ctx.organization.id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + if not connection.token: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail="Connection has no token configured.", + ) + + from app.services.forgejo_client import ForgejoClientError, get_forgejo_client + + try: + async with get_forgejo_client(connection) as client: + repos = await client.list_user_repos() + except ForgejoClientError as e: + raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to fetch repositories: {e}", + ) + + return [ + { + "full_name": r.get("full_name", ""), + "name": r.get("name", ""), + "owner": (r.get("owner") or {}).get("login", ""), + "default_branch": r.get("default_branch", "main") or "main", + "description": r.get("description") or None, + "private": bool(r.get("private", False)), + "html_url": r.get("html_url", ""), + } + for r in repos + if r.get("full_name") and r.get("name") + ] + + @router.post( "/{connection_id}/validate", response_model=ForgejoConnectionValidationResponse, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index a5b7f1f..184dfe8 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -81,6 +81,11 @@ class Settings(BaseSettings): rq_dispatch_retry_base_seconds: float = 10.0 rq_dispatch_retry_max_seconds: float = 120.0 + # Forgejo issue cache scheduled sync. + forgejo_sync_enabled: bool = True + forgejo_sync_interval_seconds: float = 60 * 60 + forgejo_sync_startup_delay_seconds: float = 60.0 + # OpenClaw gateway runtime compatibility gateway_min_version: str = "2026.02.9" diff --git a/backend/app/main.py b/backend/app/main.py index f8cd2fd..90e8db7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -46,6 +46,7 @@ from app.core.rate_limit_backend import RateLimitBackend from app.core.security_headers import SecurityHeadersMiddleware from app.db.session import init_db from app.schemas.health import HealthStatusResponse +from app.services.forgejo_sync_queue import seed_scheduled_forgejo_sync if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -468,6 +469,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: logger.info("app.lifecycle.rate_limit backend=redis") else: logger.info("app.lifecycle.rate_limit backend=memory") + seed_scheduled_forgejo_sync() logger.info("app.lifecycle.started") try: yield diff --git a/backend/app/services/forgejo_client.py b/backend/app/services/forgejo_client.py index c0be5bb..9862ad9 100644 --- a/backend/app/services/forgejo_client.py +++ b/backend/app/services/forgejo_client.py @@ -157,6 +157,30 @@ class ForgejoAPIClient: response.raise_for_status() return response.json() + async def list_user_repos( + self, + limit: int = 50, + page: int = 1, + ) -> list[dict[str, object]]: + """ + List repositories accessible to the authenticated token. + + Uses /api/v1/repos/search which returns repos the token can access, + including private repos and org repos. + + Returns: + List of repository dicts with name, full_name, owner, default_branch, etc. + """ + client = await self._get_client() + params = {"limit": limit, "page": page, "token": ""} + response = await client.get("/api/v1/repos/search", params={"limit": limit, "page": page}) + response.raise_for_status() + data = response.json() + # Forgejo /repos/search returns {"data": [...], "ok": true} + if isinstance(data, dict): + return list(data.get("data", [])) + return list(data) + def get_forgejo_client( connection: object, diff --git a/backend/app/services/forgejo_sync_queue.py b/backend/app/services/forgejo_sync_queue.py new file mode 100644 index 0000000..e631a68 --- /dev/null +++ b/backend/app/services/forgejo_sync_queue.py @@ -0,0 +1,147 @@ +"""Queue task helpers for scheduled Forgejo issue cache sync.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlmodel import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.core.config import settings +from app.core.logging import get_logger +from app.core.time import utcnow +from app.db import crud +from app.db.session import async_session_maker +from app.models.forgejo_repositories import ForgejoRepository +from app.services.forgejo_issue_sync import IssueSyncService +from app.services.queue import QueuedTask, enqueue_task_with_delay +from app.services.queue import requeue_if_failed as generic_requeue_if_failed + +logger = get_logger(__name__) +TASK_TYPE = "forgejo_issue_sync_all" +_SINGLETON_CREATED_AT = datetime(1970, 1, 1, tzinfo=UTC) + + +def _task() -> QueuedTask: + return QueuedTask( + task_type=TASK_TYPE, + payload={"scope": "active_repositories"}, + created_at=_SINGLETON_CREATED_AT, + ) + + +def _sync_enabled() -> bool: + return settings.forgejo_sync_enabled and settings.forgejo_sync_interval_seconds > 0 + + +def enqueue_forgejo_issue_sync(*, delay_seconds: float) -> bool: + """Schedule the singleton Forgejo issue sync task.""" + if not _sync_enabled(): + logger.info("forgejo.sync.queue.disabled") + return False + delay = max(0.0, float(delay_seconds)) + ok = enqueue_task_with_delay( + _task(), + settings.rq_queue_name, + delay_seconds=delay, + redis_url=settings.rq_redis_url, + ) + if ok: + logger.info( + "forgejo.sync.queue.enqueued", + extra={"delay_seconds": delay}, + ) + return ok + + +def seed_scheduled_forgejo_sync() -> bool: + """Seed periodic Forgejo sync when the API process starts.""" + return enqueue_forgejo_issue_sync( + delay_seconds=settings.forgejo_sync_startup_delay_seconds, + ) + + +def schedule_next_forgejo_issue_sync() -> bool: + """Schedule the next periodic Forgejo issue sync run.""" + return enqueue_forgejo_issue_sync( + delay_seconds=settings.forgejo_sync_interval_seconds, + ) + + +async def sync_active_forgejo_repositories(session: AsyncSession) -> dict[str, int]: + """Sync all active Forgejo repositories and return aggregate counts.""" + repositories = ( + await session.exec( + select(ForgejoRepository) + .where(ForgejoRepository.active.is_(True)) + .order_by(ForgejoRepository.created_at.asc()) + ) + ).all() + totals = { + "repositories": len(repositories), + "succeeded": 0, + "failed": 0, + "created": 0, + "updated": 0, + "open": 0, + "closed": 0, + "total": 0, + } + for repository in repositories: + try: + result = await IssueSyncService( + session=session, + organization_id=repository.organization_id, + ).sync_repository_issues(repository_id=repository.id) + except Exception as exc: + repository.last_sync_error = str(exc) + repository.updated_at = utcnow() + await crud.save(session, repository) + totals["failed"] += 1 + logger.warning( + "forgejo.sync.repository_failed", + extra={ + "repository_id": str(repository.id), + "owner": repository.owner, + "repo": repository.repo, + "error": str(exc), + }, + ) + continue + + totals["succeeded"] += 1 + for key in ("created", "updated", "open", "closed", "total"): + totals[key] += int(result.get(key, 0)) + return totals + + +async def process_forgejo_sync_queue_task(task: QueuedTask) -> None: + """Worker entrypoint for scheduled Forgejo issue sync.""" + if task.task_type not in {TASK_TYPE, "legacy"}: + raise ValueError(f"Unexpected task_type={task.task_type!r}; expected {TASK_TYPE!r}") + if not _sync_enabled(): + logger.info("forgejo.sync.skipped_disabled") + return + + try: + async with async_session_maker() as session: + totals = await sync_active_forgejo_repositories(session) + logger.info( + "forgejo.sync.completed", + extra={f"sync_{key}": value for key, value in totals.items()}, + ) + except Exception as exc: + logger.exception("forgejo.sync.failed", extra={"error": str(exc)}) + finally: + schedule_next_forgejo_issue_sync() + + +def requeue_forgejo_sync_queue_task(task: QueuedTask, *, delay_seconds: float = 0) -> bool: + """Requeue a failed Forgejo sync task with capped retries.""" + return generic_requeue_if_failed( + task, + settings.rq_queue_name, + max_retries=settings.rq_dispatch_max_retries, + redis_url=settings.rq_redis_url, + delay_seconds=max(0.0, delay_seconds), + ) diff --git a/backend/app/services/queue_worker.py b/backend/app/services/queue_worker.py index 61eb033..c529cbf 100644 --- a/backend/app/services/queue_worker.py +++ b/backend/app/services/queue_worker.py @@ -9,6 +9,11 @@ from dataclasses import dataclass from app.core.config import settings from app.core.logging import get_logger +from app.services.forgejo_sync_queue import TASK_TYPE as FORGEJO_SYNC_TASK_TYPE +from app.services.forgejo_sync_queue import ( + process_forgejo_sync_queue_task, + requeue_forgejo_sync_queue_task, +) from app.services.openclaw.lifecycle_queue import TASK_TYPE as LIFECYCLE_RECONCILE_TASK_TYPE from app.services.openclaw.lifecycle_queue import ( requeue_lifecycle_queue_task, @@ -49,6 +54,17 @@ _TASK_HANDLERS: dict[str, _TaskHandler] = { ), requeue=lambda task, delay: requeue_webhook_queue_task(task, delay_seconds=delay), ), + FORGEJO_SYNC_TASK_TYPE: _TaskHandler( + handler=process_forgejo_sync_queue_task, + attempts_to_delay=lambda attempts: min( + settings.rq_dispatch_retry_base_seconds * (2 ** max(0, attempts)), + settings.rq_dispatch_retry_max_seconds, + ), + requeue=lambda task, delay: requeue_forgejo_sync_queue_task( + task, + delay_seconds=delay, + ), + ), } diff --git a/backend/tests/test_forgejo_client.py b/backend/tests/test_forgejo_client.py index 1a0fe88..f9dfe14 100644 --- a/backend/tests/test_forgejo_client.py +++ b/backend/tests/test_forgejo_client.py @@ -39,14 +39,9 @@ def test_get_forgejo_client_factory() -> None: base_url = "https://forgejo.example.com" token = "ghp_testtoken123" - import asyncio - - async def test(): - client = await get_forgejo_client(MockConnection()) - assert client.base_url == "https://forgejo.example.com" - assert client.token == "ghp_testtoken123" - - asyncio.run(test()) + client = get_forgejo_client(MockConnection()) + assert client.base_url == "https://forgejo.example.com" + assert client.token == "ghp_testtoken123" def test_get_forgejo_client_with_api_path() -> None: @@ -54,10 +49,5 @@ def test_get_forgejo_client_with_api_path() -> None: class MockConnection: base_url = "https://forgejo.example.com/api/v1" - import asyncio - - async def test(): - client = await get_forgejo_client(MockConnection()) - assert client.base_url == "https://forgejo.example.com" - - asyncio.run(test()) + client = get_forgejo_client(MockConnection()) + assert client.base_url == "https://forgejo.example.com" diff --git a/backend/tests/test_forgejo_sync_queue.py b/backend/tests/test_forgejo_sync_queue.py new file mode 100644 index 0000000..54fdf23 --- /dev/null +++ b/backend/tests/test_forgejo_sync_queue.py @@ -0,0 +1,230 @@ +# ruff: noqa: INP001 +"""Scheduled Forgejo issue sync queue tests.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine +from sqlmodel import SQLModel, col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from app import models as _models +from app.models.forgejo_connections import ForgejoConnection +from app.models.forgejo_repositories import ForgejoRepository +from app.models.organizations import Organization +from app.services import forgejo_sync_queue +from app.services.forgejo_sync_queue import ( + enqueue_forgejo_issue_sync, + process_forgejo_sync_queue_task, + sync_active_forgejo_repositories, +) +from app.services.queue import QueuedTask + +_MODEL_REGISTRY = _models + + +async def _make_engine() -> AsyncEngine: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + return engine + + +async def _seed_repositories( + session: AsyncSession, +) -> tuple[ForgejoRepository, ForgejoRepository, ForgejoRepository]: + organization_id = uuid4() + connection_id = uuid4() + session.add(Organization(id=organization_id, name=f"org-{organization_id}")) + session.add( + ForgejoConnection( + id=connection_id, + organization_id=organization_id, + name="Forgejo", + base_url="https://forgejo.example.local", + ) + ) + active_repo = ForgejoRepository( + id=uuid4(), + organization_id=organization_id, + connection_id=connection_id, + owner="openclaw", + repo="pipeline", + active=True, + ) + failing_repo = ForgejoRepository( + id=uuid4(), + organization_id=organization_id, + connection_id=connection_id, + owner="openclaw", + repo="broken", + active=True, + ) + inactive_repo = ForgejoRepository( + id=uuid4(), + organization_id=organization_id, + connection_id=connection_id, + owner="openclaw", + repo="inactive", + active=False, + ) + session.add(active_repo) + session.add(failing_repo) + session.add(inactive_repo) + await session.commit() + return active_repo, failing_repo, inactive_repo + + +def test_enqueue_forgejo_issue_sync_uses_singleton_delayed_task( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def _fake_enqueue_with_delay( + task: QueuedTask, + queue_name: str, + *, + delay_seconds: float, + redis_url: str | None = None, + ) -> bool: + captured["task"] = task + captured["queue_name"] = queue_name + captured["delay_seconds"] = delay_seconds + captured["redis_url"] = redis_url + return True + + monkeypatch.setattr(forgejo_sync_queue.settings, "forgejo_sync_enabled", True) + monkeypatch.setattr(forgejo_sync_queue.settings, "forgejo_sync_interval_seconds", 3600) + monkeypatch.setattr( + "app.services.forgejo_sync_queue.enqueue_task_with_delay", + _fake_enqueue_with_delay, + ) + + assert enqueue_forgejo_issue_sync(delay_seconds=30) is True + task = captured["task"] + assert isinstance(task, QueuedTask) + assert task.task_type == "forgejo_issue_sync_all" + assert task.payload == {"scope": "active_repositories"} + assert task.created_at.isoformat() == "1970-01-01T00:00:00+00:00" + assert captured["delay_seconds"] == 30 + + +def test_enqueue_forgejo_issue_sync_respects_disabled_setting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(forgejo_sync_queue.settings, "forgejo_sync_enabled", False) + monkeypatch.setattr( + "app.services.forgejo_sync_queue.enqueue_task_with_delay", + lambda *args, **kwargs: pytest.fail("disabled sync should not enqueue"), + ) + + assert enqueue_forgejo_issue_sync(delay_seconds=30) is False + + +@pytest.mark.asyncio +async def test_sync_active_forgejo_repositories_syncs_active_and_records_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + + class _FakeIssueSyncService: + def __init__(self, *, session: AsyncSession, organization_id: object) -> None: + del session, organization_id + + async def sync_repository_issues(self, *, repository_id: object) -> dict[str, int]: + calls.append(str(repository_id)) + if repository_id == failing_repo.id: + raise RuntimeError("Forgejo unavailable") + return {"created": 2, "updated": 3, "open": 4, "closed": 1, "total": 5} + + engine = await _make_engine() + session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + try: + async with session_maker() as session: + active_repo, failing_repo, inactive_repo = await _seed_repositories(session) + monkeypatch.setattr( + "app.services.forgejo_sync_queue.IssueSyncService", + _FakeIssueSyncService, + ) + + totals = await sync_active_forgejo_repositories(session) + + assert totals == { + "repositories": 2, + "succeeded": 1, + "failed": 1, + "created": 2, + "updated": 3, + "open": 4, + "closed": 1, + "total": 5, + } + assert calls == [str(active_repo.id), str(failing_repo.id)] + + async with session_maker() as session: + stored_failing_repo = ( + await session.exec( + select(ForgejoRepository).where(col(ForgejoRepository.id) == failing_repo.id) + ) + ).one() + stored_inactive_repo = ( + await session.exec( + select(ForgejoRepository).where(col(ForgejoRepository.id) == inactive_repo.id) + ) + ).one() + assert stored_failing_repo.last_sync_error == "Forgejo unavailable" + assert stored_inactive_repo.last_sync_error is None + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_process_forgejo_sync_queue_task_reschedules_after_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + called: list[str] = [] + + async def _fake_sync_active_repositories(session: AsyncSession) -> dict[str, int]: + del session + called.append("sync") + return { + "repositories": 0, + "succeeded": 0, + "failed": 0, + "created": 0, + "updated": 0, + "open": 0, + "closed": 0, + "total": 0, + } + + class _FakeSessionMaker: + async def __aenter__(self) -> AsyncSession: + return None # type: ignore[return-value] + + async def __aexit__(self, *args: object) -> None: + return None + + monkeypatch.setattr(forgejo_sync_queue.settings, "forgejo_sync_enabled", True) + monkeypatch.setattr(forgejo_sync_queue.settings, "forgejo_sync_interval_seconds", 12) + monkeypatch.setattr("app.services.forgejo_sync_queue.async_session_maker", _FakeSessionMaker) + monkeypatch.setattr( + "app.services.forgejo_sync_queue.sync_active_forgejo_repositories", + _fake_sync_active_repositories, + ) + monkeypatch.setattr( + "app.services.forgejo_sync_queue.schedule_next_forgejo_issue_sync", + lambda: called.append("reschedule") or True, + ) + + await process_forgejo_sync_queue_task( + QueuedTask( + task_type="forgejo_issue_sync_all", + payload={"scope": "active_repositories"}, + created_at=forgejo_sync_queue._SINGLETON_CREATED_AT, + ) + ) + + assert called == ["sync", "reschedule"] diff --git a/compose.yml b/compose.yml index 42a6336..a458c7f 100644 --- a/compose.yml +++ b/compose.yml @@ -92,6 +92,9 @@ services: RQ_QUEUE_NAME: ${RQ_QUEUE_NAME:-default} RQ_DISPATCH_THROTTLE_SECONDS: ${RQ_DISPATCH_THROTTLE_SECONDS:-2.0} RQ_DISPATCH_MAX_RETRIES: ${RQ_DISPATCH_MAX_RETRIES:-3} + FORGEJO_SYNC_ENABLED: ${FORGEJO_SYNC_ENABLED:-true} + FORGEJO_SYNC_INTERVAL_SECONDS: ${FORGEJO_SYNC_INTERVAL_SECONDS:-3600} + FORGEJO_SYNC_STARTUP_DELAY_SECONDS: ${FORGEJO_SYNC_STARTUP_DELAY_SECONDS:-60} restart: unless-stopped volumes: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index eb2ffba..6e957e8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -42,6 +42,21 @@ See `.env.example` for defaults and required values. - **Purpose:** Comma-separated list of trusted reverse-proxy IPs or CIDRs used to honor `Forwarded` / `X-Forwarded-For` client IP headers. - **Gotcha:** Leave this blank unless the direct peer is a proxy you control. +### `FORGEJO_SYNC_ENABLED` + +- **Default:** `true` +- **Purpose:** Enables the scheduled RQ task that syncs all active Forgejo repository issue caches. + +### `FORGEJO_SYNC_INTERVAL_SECONDS` + +- **Default:** `3600` +- **Purpose:** Interval between scheduled Forgejo issue-cache sync runs. + +### `FORGEJO_SYNC_STARTUP_DELAY_SECONDS` + +- **Default:** `60` +- **Purpose:** Delay before the first scheduled Forgejo sync task is seeded after API startup. + ## Security response headers These environment variables control security headers added to every API response. Set any variable to blank (`""`) to disable the corresponding header. diff --git a/frontend/src/app/git-projects/repositories/[repositoryId]/edit/page.tsx b/frontend/src/app/git-projects/repositories/[repositoryId]/edit/page.tsx index 3823852..3c492a8 100644 --- a/frontend/src/app/git-projects/repositories/[repositoryId]/edit/page.tsx +++ b/frontend/src/app/git-projects/repositories/[repositoryId]/edit/page.tsx @@ -130,6 +130,7 @@ export default function ForgejoRepositoriesEditPage({ repo: repository.repo, display_name: repository.display_name, default_branch: repository.default_branch, + showWebhookSecret: true, }; return ( diff --git a/frontend/src/app/settings/git-projects/page.tsx b/frontend/src/app/settings/git-projects/page.tsx new file mode 100644 index 0000000..a7dc285 --- /dev/null +++ b/frontend/src/app/settings/git-projects/page.tsx @@ -0,0 +1,688 @@ +"use client"; + +export const dynamic = "force-dynamic"; + +import { useCallback, useEffect, useMemo, useState } from "react"; +import type { ReactNode } from "react"; +import Link from "next/link"; +import { + AlertCircle, + CheckCircle2, + CircleDot, + Clock, + Copy, + ExternalLink, + GitBranch, + KeyRound, + Link2, + RefreshCw, + Server, + Webhook, +} from "lucide-react"; + +import { useAuth } from "@/auth/clerk"; +import { ForgejoConnectionsTable } from "@/components/git/ForgejoConnectionsTable"; +import { ForgejoRepositoriesTable } from "@/components/git/ForgejoRepositoriesTable"; +import { DashboardPageLayout } from "@/components/templates/DashboardPageLayout"; +import { Button } from "@/components/ui/button"; +import { ConfirmActionDialog } from "@/components/ui/confirm-action-dialog"; +import { getApiBaseUrl } from "@/lib/api-base"; +import { + deleteForgejoConnection, + deleteForgejoRepository, + getForgejoConnections, + getForgejoRepositories, + syncRepository, + validateConnection, + validateRepository, + type ForgejoConnection, + type ForgejoRepository, +} from "@/lib/api-forgejo"; + +type Notice = { + tone: "success" | "error"; + message: string; +}; + +type DeleteTarget = + | { type: "connection"; item: ForgejoConnection } + | { type: "repository"; item: ForgejoRepository }; + +const repositoryName = (repository: ForgejoRepository) => + repository.display_name || `${repository.owner}/${repository.repo}`; + +const formatTimestamp = (value: string | null) => { + if (!value) return "Never"; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value; + return date.toLocaleString(); +}; + +function NoticeBanner({ notice }: { notice: Notice }) { + return ( +
+ {notice.tone === "success" ? ( + + ) : ( + + )} + {notice.message} +
+ ); +} + +function StatCard({ + icon, + label, + value, + caption, +}: { + icon: ReactNode; + label: string; + value: string; + caption: string; +}) { + return ( +
+
+
+

+ {label} +

+

+ {value} +

+
+
+ {icon} +
+
+

{caption}

+
+ ); +} + +function CopyButton({ value, label }: { value: string; label?: string }) { + const [copied, setCopied] = useState(false); + + const handleCopy = async () => { + try { + await navigator.clipboard.writeText(value); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + // Clipboard API unavailable — silent fail + } + }; + + return ( + + ); +} + +export default function GitProjectSettingsPage() { + const auth = useAuth(); + const [connections, setConnections] = useState([]); + const [repositories, setRepositories] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [isSyncingAll, setIsSyncingAll] = useState(false); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); + const [deleteTarget, setDeleteTarget] = useState(null); + const [deleteError, setDeleteError] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + + // Auto-dismiss notices after 8s + useEffect(() => { + if (!notice) return; + const id = setTimeout(() => setNotice(null), 8000); + return () => clearTimeout(id); + }, [notice]); + + const loadSettings = useCallback(async () => { + try { + setIsLoading(true); + const [connectionsData, repositoriesData] = await Promise.all([ + getForgejoConnections(), + getForgejoRepositories(), + ]); + setConnections(connectionsData); + setRepositories(repositoriesData); + setError(null); + } catch (err) { + setError( + err instanceof Error + ? err.message + : "Pipeline could not load Git Project settings.", + ); + } finally { + setIsLoading(false); + } + }, []); + + useEffect(() => { + if (auth.isSignedIn) { + loadSettings(); + } + }, [auth.isSignedIn, loadSettings]); + + const activeConnections = useMemo( + () => connections.filter((c) => c.active).length, + [connections], + ); + const activeRepositories = useMemo( + () => repositories.filter((r) => r.active).length, + [repositories], + ); + const latestSync = useMemo(() => { + const synced = repositories + .map((r) => r.last_sync_at) + .filter((v): v is string => Boolean(v)) + .map((v) => new Date(v)) + .filter((d) => !Number.isNaN(d.getTime())); + if (synced.length === 0) return null; + return synced.reduce((latest, d) => + d.getTime() > latest.getTime() ? d : latest, + ); + }, [repositories]); + const syncErrorCount = useMemo( + () => repositories.filter((r) => r.last_sync_error).length, + [repositories], + ); + + const webhookBaseUrl = useMemo(() => { + try { + return getApiBaseUrl(); + } catch { + return ""; + } + }, []); + + const handleValidateConnection = async (connection: ForgejoConnection) => { + try { + const result = await validateConnection(connection.id); + setNotice( + result.status.ok + ? { + tone: "success", + message: `"${connection.name}" validated in ${Math.round(result.response_time_ms)}ms.`, + } + : { + tone: "error", + message: `Connection validation failed: ${result.status.error_message ?? "Unknown error"}`, + }, + ); + return result; + } catch (err) { + setNotice({ + tone: "error", + message: + err instanceof Error ? err.message : "Failed to validate connection.", + }); + throw err; + } + }; + + const handleValidateRepository = async (repository: ForgejoRepository) => { + try { + const result = await validateRepository(repository.id); + setNotice( + result.status.ok + ? { + tone: "success", + message: `${repositoryName(repository)} is reachable from Pipeline.`, + } + : { + tone: "error", + message: `Repository validation failed: ${result.status.error_message ?? "Unknown error"}`, + }, + ); + return result; + } catch (err) { + setNotice({ + tone: "error", + message: + err instanceof Error + ? err.message + : "Failed to validate repository.", + }); + throw err; + } + }; + + const handleSyncRepository = async (repository: ForgejoRepository) => { + try { + const result = await syncRepository(repository.id); + setNotice({ + tone: "success", + message: `${repositoryName(repository)} synced: ${result.created} created, ${result.updated} updated, ${result.open} open, ${result.closed} closed.`, + }); + const data = await getForgejoRepositories(); + setRepositories(data); + return result; + } catch (err) { + setNotice({ + tone: "error", + message: + err instanceof Error ? err.message : "Failed to sync repository.", + }); + throw err; + } + }; + + const handleSyncAll = async () => { + const active = repositories.filter((r) => r.active); + if (active.length === 0) return; + setIsSyncingAll(true); + let succeeded = 0; + let failed = 0; + await Promise.allSettled( + active.map(async (repo) => { + try { + await syncRepository(repo.id); + succeeded++; + } catch { + failed++; + } + }), + ); + const data = await getForgejoRepositories(); + setRepositories(data); + setIsSyncingAll(false); + setNotice( + failed === 0 + ? { + tone: "success", + message: `All ${succeeded} repositories synced successfully.`, + } + : { + tone: "error", + message: `Sync completed: ${succeeded} succeeded, ${failed} failed.`, + }, + ); + }; + + const confirmDelete = async () => { + if (!deleteTarget) return; + setIsDeleting(true); + setDeleteError(null); + try { + if (deleteTarget.type === "connection") { + await deleteForgejoConnection(deleteTarget.item.id); + setConnections((prev) => + prev.filter((c) => c.id !== deleteTarget.item.id), + ); + setNotice({ + tone: "success", + message: `Deleted "${deleteTarget.item.name}".`, + }); + } else { + await deleteForgejoRepository(deleteTarget.item.id); + setRepositories((prev) => + prev.filter((r) => r.id !== deleteTarget.item.id), + ); + setNotice({ + tone: "success", + message: `Deleted "${repositoryName(deleteTarget.item)}".`, + }); + } + setDeleteTarget(null); + } catch (err) { + setDeleteError( + err instanceof Error + ? err.message + : "Pipeline could not delete this item.", + ); + } finally { + setIsDeleting(false); + } + }; + + return ( + <> + + + + + + + + } + > +
+ {notice ? : null} + {error ? ( +
+ + {error} +
+ ) : null} + + {/* Stats */} +
+ } + label="Connections" + value={`${activeConnections}/${connections.length}`} + caption="Active Forgejo connection records." + /> + } + label="Repositories" + value={`${activeRepositories}/${repositories.length}`} + caption="Active tracked repositories." + /> + } + label="Latest Sync" + value={formatTimestamp(latestSync?.toISOString() ?? null)} + caption="Most recent repository sync timestamp." + /> + } + label="Sync Errors" + value={String(syncErrorCount)} + caption="Repositories with a recorded sync error." + /> +
+ + {/* Connections */} +
+
+
+

+ Forgejo Connections +

+

+ URL and token records used by tracked repositories. +

+
+ + + +
+
+ + setDeleteTarget({ type: "connection", item: connection }) + } + onValidate={handleValidateConnection} + /> +
+
+ + {/* Repositories */} +
+
+
+

+ Tracked Repositories +

+

+ Repositories whose issues are cached and shown in Pipeline. +

+
+
+ + + + + + +
+
+
+ + setDeleteTarget({ type: "repository", item: repository }) + } + onSync={handleSyncRepository} + onValidate={handleValidateRepository} + /> +
+
+ + {/* Webhook Setup */} +
+
+
+ +
+
+

+ Webhook Setup +

+

+ Configure webhooks in Forgejo to push issue updates to + Pipeline in real time, without waiting for the scheduled sync. +

+
+
+ + {repositories.length === 0 ? ( +

+ No repositories tracked yet. Add a repository to see webhook + URLs. +

+ ) : ( +
+

+ In each Forgejo repository, go to{" "} + + Settings → Webhooks → Add Webhook + + . Set the target URL below, content type{" "} + + application/json + + , and paste a secret. Store the same secret on the repository + record in Pipeline via{" "} + + Edit Repository + + . +

+
+ + + + + + + + + + {repositories.map((repo) => { + const webhookUrl = webhookBaseUrl + ? `${webhookBaseUrl}/api/v1/forgejo/webhooks/${repo.id}` + : `…/api/v1/forgejo/webhooks/${repo.id}`; + return ( + + + + + + ); + })} + +
+ Repository + + Webhook URL + + Secret +
+ + {repositoryName(repo)} + + + {repo.active ? "" : "(inactive)"} + + +
+ + {webhookUrl} + + +
+ 		+ {repo.has_webhook_secret ? ( + + + Configured + + ) : ( + + + Not set + + )} +
+
+
+ )} +
+ + {/* Scheduled Sync */} +
+
+
+ +
+
+

+ Scheduled Sync +

+

+ Pipeline runs a background sync for all active repositories + every 60 minutes. + This keeps issues current without manual syncing or webhooks. + The interval is configured via environment variable and cannot + be changed from the UI. +

+
+ + Env:{" "} + + FORGEJO_SYNC_ENABLED + + + + Env:{" "} + + FORGEJO_SYNC_INTERVAL_SECONDS + + +
+
+
+
+
+ + + { + if (!open) setDeleteTarget(null); + }} + title={ + deleteTarget?.type === "connection" + ? "Delete Git Project connection" + : "Delete Git Project repository" + } + description={ + deleteTarget?.type === "connection" + ? `Delete "${deleteTarget.item.name}" from Pipeline? Repositories that use this connection will stop syncing.` + : deleteTarget?.type === "repository" + ? `Delete "${repositoryName(deleteTarget.item)}" from Pipeline? Synced issue records for this repository will be removed.` + : "" + } + onConfirm={confirmDelete} + isConfirming={isDeleting} + errorMessage={deleteError} + confirmLabel="Delete" + confirmingLabel="Deleting…" + confirmClassName="bg-[color:var(--danger)] text-white hover:bg-[color:var(--danger)]/90" + cancelLabel="Keep" + /> + + ); +} diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index d93b19f..d37add2 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -3,11 +3,21 @@ export const dynamic = "force-dynamic"; import { useMemo, useState } from "react"; +import Link from "next/link"; import { useRouter } from "next/navigation"; import { useAuth, useUser } from "@/auth/clerk"; import { useQueryClient } from "@tanstack/react-query"; -import { Globe, Mail, RotateCcw, Save, Trash2, User } from "lucide-react"; +import { + ArrowRight, + GitBranch, + Globe, + Mail, + RotateCcw, + Save, + Trash2, + User, +} from "lucide-react"; import { useDeleteMeApiV1UsersMeDelete, @@ -241,6 +251,27 @@ export default function SettingsPage() { +
+
+
+

+ + Git Projects +

+

+ Manage Forgejo connections, tracked repositories, and issue + sync. +

+
+ + + +
+
+

Delete account diff --git a/frontend/src/components/git/ForgejoRepositoryForm.tsx b/frontend/src/components/git/ForgejoRepositoryForm.tsx index 3ff27dd..0482cd3 100644 --- a/frontend/src/components/git/ForgejoRepositoryForm.tsx +++ b/frontend/src/components/git/ForgejoRepositoryForm.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from "react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; -import { Loader2 } from "lucide-react"; +import { KeyRound, Loader2, Lock, Unlock } from "lucide-react"; import { Select, SelectContent, @@ -14,8 +14,10 @@ import { } from "@/components/ui/select"; import { + getConnectionRepos, getForgejoConnections, type ForgejoConnection, + type ForgejoRemoteRepo, type ForgejoRepositoryCreate, } from "@/lib/api-forgejo"; @@ -34,6 +36,9 @@ export interface ForgejoRepositoryFormValues { repo: string; display_name?: string; default_branch?: string; + webhook_secret?: string; + /** When true, shows the webhook secret field (edit mode only) */ + showWebhookSecret?: boolean; } export function ForgejoRepositoryForm({ @@ -57,21 +62,35 @@ export function ForgejoRepositoryForm({ const [defaultBranch, setDefaultBranch] = useState( defaultValues.default_branch || "main", ); + const [webhookSecret, setWebhookSecret] = useState( + defaultValues.webhook_secret || "", + ); + const showWebhookSecret = Boolean(defaultValues.showWebhookSecret); - // Get connections for dropdown const [connections, setConnections] = useState([]); - const [isLoadingConnections, setIsLoadingConnections] = useState(false); + + // Remote repo discovery + const [remoteRepos, setRemoteRepos] = useState([]); + const [isLoadingRepos, setIsLoadingRepos] = useState(false); + const [repoLoadError, setRepoLoadError] = useState(null); + const [selectedFullName, setSelectedFullName] = useState( + defaultValues.owner && defaultValues.repo + ? `${defaultValues.owner}/${defaultValues.repo}` + : "", + ); + const isBusy = isSubmitting || isSaving; + const isEditMode = Boolean(defaultValues.connection_id); useEffect(() => { const fetchConnections = async () => { try { setIsLoadingConnections(true); const data = await getForgejoConnections(); - setConnections(data.filter((connection) => connection.active)); + setConnections(data.filter((c) => c.active)); } catch { - setError("Failed to load connections"); + setError("Failed to load connections."); } finally { setIsLoadingConnections(false); } @@ -79,16 +98,55 @@ export function ForgejoRepositoryForm({ fetchConnections(); }, []); + // When a connection is chosen, fetch accessible repos from that Forgejo instance + useEffect(() => { + if (!connectionId || isEditMode) return; + setRemoteRepos([]); + setSelectedFullName(""); + setOwner(""); + setRepo(""); + setDefaultBranch("main"); + setRepoLoadError(null); + + const fetchRepos = async () => { + setIsLoadingRepos(true); + try { + const data = await getConnectionRepos(connectionId); + setRemoteRepos(data); + } catch (err) { + setRepoLoadError( + err instanceof Error + ? err.message + : "Could not fetch repositories from this connection.", + ); + } finally { + setIsLoadingRepos(false); + } + }; + fetchRepos(); + }, [connectionId, isEditMode]); + + const handleRepoSelect = (fullName: string) => { + setSelectedFullName(fullName); + const found = remoteRepos.find((r) => r.full_name === fullName); + if (!found) return; + setOwner(found.owner); + setRepo(found.name); + setDefaultBranch(found.default_branch || "main"); + if (!displayName) setDisplayName(found.name); + }; + async function handleSubmit(e: React.FormEvent) { e.preventDefault(); setError(null); if (!connectionId) { - setError( - "Choose a Git Projects connection before saving this repository.", - ); + setError("Choose a connection before saving."); + return; + } + if (!owner || !repo) { + setError("Select a repository or enter an owner and repository name."); return; } - try { setIsSaving(true); await onSubmit({ @@ -97,9 +155,12 @@ export function ForgejoRepositoryForm({ repo, display_name: displayName || undefined, default_branch: defaultBranch, + ...(showWebhookSecret + ? { webhook_secret: webhookSecret || null } + : {}), }); } catch (err) { - setError(err instanceof Error ? err.message : "An error occurred"); + setError(err instanceof Error ? err.message : "An error occurred."); } finally { setIsSaving(false); } @@ -125,6 +186,7 @@ export function ForgejoRepositoryForm({ )} + {/* Connection */}
-
+ {/* Repo picker — shown when connection is selected in create mode */} + {!isEditMode && connectionId && (
- - setOwner(e.target.value)} - placeholder="openclaw" - disabled={isBusy} - required - /> -

- The owner of the repository (username or organization). -

-
- -
-
-
+ )} + {/* Manual owner/repo fields — always shown in edit mode */} + {isEditMode && ( + + )} + + {/* Display Name */}

- A friendly name for this repository. If not provided, the owner/repo - will be used. + Friendly name shown in Pipeline. Defaults to owner/repo if blank.

+ {/* Default Branch */}
+ + {/* Webhook Secret — edit mode only */} + {showWebhookSecret && ( +
+ + setWebhookSecret(e.target.value)} + placeholder="Leave blank to keep existing secret" + disabled={isBusy} + autoComplete="new-password" + /> +

+ Secret used to verify incoming Forgejo webhook payloads. Set the + same value in your Forgejo repository webhook settings. Leave + blank to keep the current secret, or clear and save to remove it. +

+
+ )}
@@ -251,3 +403,69 @@ export function ForgejoRepositoryForm({ ); } + +function ManualOwnerRepoFields({ + owner, + repo, + onOwnerChange, + onRepoChange, + disabled, +}: { + owner: string; + repo: string; + onOwnerChange: (v: string) => void; + onRepoChange: (v: string) => void; + disabled: boolean; +}) { + return ( +
+
+ + onOwnerChange(e.target.value)} + placeholder="myorg" + disabled={disabled} + required + /> +

Username or organization name.

+
+
+ + onRepoChange(e.target.value)} + placeholder="my-project" + disabled={disabled} + required + /> +

Repository name.

+
+
+ ); +} + +// Filter/toggle helpers re-exported for table usage +export function RepositoriesTableFilter({ + value, + onChange, +}: { + value: string; + onChange: (value: string) => void; +}) { + return ( + onChange(e.target.value)} + className="h-9 w-[150px] rounded-xl border border-[color:var(--border)] bg-[color:var(--surface)] px-3 py-1 text-sm text-strong focus:border-[color:var(--accent)] focus:outline-none focus:ring-2 focus:ring-[color:var(--accent)] lg:w-[250px]" + /> + ); +} diff --git a/frontend/src/components/organisms/DashboardSidebar.tsx b/frontend/src/components/organisms/DashboardSidebar.tsx index 8d6180d..02b9ba7 100644 --- a/frontend/src/components/organisms/DashboardSidebar.tsx +++ b/frontend/src/components/organisms/DashboardSidebar.tsx @@ -199,6 +199,13 @@ export function DashboardSidebar() { Organization + + + Settings + {isAdmin ? ( = { @@ -159,6 +161,25 @@ export async function deleteForgejoRepository( ); } +// Remote repo discovery +export interface ForgejoRemoteRepo { + full_name: string; + name: string; + owner: string; + default_branch: string; + description: string | null; + private: boolean; + html_url: string; +} + +export async function getConnectionRepos( + connectionId: string, +): Promise { + return fetchJson( + `/api/v1/forgejo/connections/${connectionId}/repos`, + ); +} + // Forgejo Sync & Validation API export async function syncRepository(repositoryId: string): Promise<{ created: number;