diff --git a/backend/app/api/provider_credentials.py b/backend/app/api/provider_credentials.py index 2161cec..2734639 100644 --- a/backend/app/api/provider_credentials.py +++ b/backend/app/api/provider_credentials.py @@ -21,6 +21,7 @@ from app.models.provider_credentials import ProviderCredential, SUPPORTED_PROVID from app.schemas.provider_credentials import ( ProviderCredentialCreate, ProviderCredentialRead, + ProviderCredentialTestRequest, ProviderCredentialUpdate, ProviderUsageLiveRead, RequestWindowRead, @@ -116,6 +117,60 @@ async def create_provider_credential( return _to_read(cred) +@router.post("/test", response_model=ProviderUsageLiveRead) +async def test_provider_credential( + payload: ProviderCredentialTestRequest, + ctx: OrganizationContext = ORG_ADMIN_DEP, +) -> ProviderUsageLiveRead: + """Validate provider credentials without saving them. Admin-only.""" + if payload.provider not in SUPPORTED_PROVIDERS: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unsupported provider '{payload.provider}'. Supported: {sorted(SUPPORTED_PROVIDERS)}", + ) + + account_key = payload.account_key.strip() or "test" + live = await fetch_provider_usage( + credential_id=f"test:{ctx.organization.id}:{payload.provider}:{account_key}", + provider=payload.provider, + account_key=account_key, + api_key=payload.api_key, + base_url=payload.base_url, + force_refresh=True, + ) + + def _tok(w) -> TokenWindowRead: + return TokenWindowRead( + limit=w.limit, + remaining=w.remaining, + used=w.used, + pct_used=w.pct_used, + reset_at=w.reset_at.isoformat() if w.reset_at else None, + reset_in_ms=w.reset_in_ms, + ) + + def _req(w) -> RequestWindowRead: + return RequestWindowRead( + limit=w.limit, + remaining=w.remaining, + reset_at=w.reset_at.isoformat() if w.reset_at else None, + reset_in_ms=w.reset_in_ms, + ) + + return ProviderUsageLiveRead( + provider=live.provider, + account_key=live.account_key, + checked_at=live.checked_at.isoformat(), + reachable=live.reachable, + error=live.error, + tokens=_tok(live.tokens), + input_tokens=_tok(live.input_tokens), + requests=_req(live.requests), + models=live.models, + debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None, + ) + + @router.get("/{credential_id}", response_model=ProviderCredentialRead) async def get_provider_credential( credential_id: UUID, @@ -210,6 +265,7 @@ async def get_provider_usage_live( input_tokens=_tok(live.input_tokens), requests=_req(live.requests), models=live.models, + debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None, ) diff --git a/backend/app/schemas/provider_credentials.py b/backend/app/schemas/provider_credentials.py index f8faba8..46dba5b 100644 --- a/backend/app/schemas/provider_credentials.py +++ b/backend/app/schemas/provider_credentials.py @@ -19,6 +19,13 @@ class ProviderCredentialCreate(SQLModel): active: bool = True +class ProviderCredentialTestRequest(SQLModel): + provider: str + account_key: str = "test" + api_key: str | None = None + base_url: str | None = None + + class ProviderCredentialUpdate(SQLModel): display_name: str | None = None api_key: str | None = None # None = keep existing; "" = clear it @@ -54,6 +61,8 @@ class ProviderUsageLiveRead(SQLModel): input_tokens: TokenWindowRead # Anthropic splits input tokens separately requests: RequestWindowRead models: list[str] = [] + # Optional debugging aid: exact rate-limit header names returned by provider. + debug_rate_limit_headers: list[str] | None = None class ProviderCredentialRead(SQLModel): diff --git a/backend/tests/test_provider_credentials_usage_api.py b/backend/tests/test_provider_credentials_usage_api.py new file mode 100644 index 0000000..3465784 --- /dev/null +++ b/backend/tests/test_provider_credentials_usage_api.py @@ -0,0 +1,189 @@ +# ruff: noqa: INP001 +"""Integration tests for provider credential live-usage API behavior.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from fastapi import APIRouter, FastAPI +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine +from sqlmodel import SQLModel +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.api.deps import require_org_admin, require_org_member +from app.api.provider_credentials import router as provider_credentials_router +from app.core.time import utcnow +from app.db.session import get_session +from app.models.organization_members import OrganizationMember +from app.models.organizations import Organization +from app.models.provider_credentials import ProviderCredential +from app.services.organizations import OrganizationContext +from app.services.provider_usage import ProviderUsageLive + + +async def _make_engine() -> AsyncEngine: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + return engine + + +def _build_test_app( + session_maker: async_sessionmaker[AsyncSession], + ctx: OrganizationContext, +) -> FastAPI: + app = FastAPI() + api_v1 = APIRouter(prefix="/api/v1") + api_v1.include_router(provider_credentials_router) + app.include_router(api_v1) + + async def _override_get_session() -> AsyncSession: + async with session_maker() as session: + yield session + + async def _override_require_org_member() -> OrganizationContext: + return ctx + + async def _override_require_org_admin() -> OrganizationContext: + return ctx + + app.dependency_overrides[get_session] = _override_get_session + app.dependency_overrides[require_org_member] = _override_require_org_member + app.dependency_overrides[require_org_admin] = _override_require_org_admin + return app + + +@pytest.mark.asyncio +async def test_usage_response_includes_rate_limit_header_names(monkeypatch: pytest.MonkeyPatch) -> None: + engine = await _make_engine() + session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + organization = Organization(id=uuid4(), name="Pipeline") + member = OrganizationMember( + id=uuid4(), + organization_id=organization.id, + user_id=uuid4(), + role="owner", + ) + app = _build_test_app( + session_maker, + OrganizationContext(organization=organization, member=member), + ) + + credential = ProviderCredential( + id=uuid4(), + organization_id=organization.id, + provider="anthropic", + account_key="Claude", + display_name="Claude", + api_key="sk-ant-test", + api_key_last_four="test", + active=True, + ) + + async def _fake_fetch_provider_usage(**_: object) -> ProviderUsageLive: + result = ProviderUsageLive( + provider="anthropic", + account_key="Claude", + checked_at=utcnow(), + reachable=True, + ) + result.raw_headers = { + "anthropic-ratelimit-requests-limit": "1000", + "anthropic-ratelimit-requests-remaining": "999", + "anthropic-ratelimit-requests-reset": "2026-05-21T12:00:00Z", + } + return result + + monkeypatch.setattr( + "app.api.provider_credentials.fetch_provider_usage", + _fake_fetch_provider_usage, + ) + + try: + async with session_maker() as session: + session.add(organization) + session.add(credential) + await session.commit() + + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://testserver", + ) as client: + response = await client.get(f"/api/v1/provider-credentials/{credential.id}/usage") + + assert response.status_code == 200 + data = response.json() + assert data["provider"] == "anthropic" + assert data["reachable"] is True + assert data["debug_rate_limit_headers"] == [ + "anthropic-ratelimit-requests-limit", + "anthropic-ratelimit-requests-remaining", + "anthropic-ratelimit-requests-reset", + ] + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_test_endpoint_returns_live_result(monkeypatch: pytest.MonkeyPatch) -> None: + engine = await _make_engine() + session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + organization = Organization(id=uuid4(), name="Pipeline") + member = OrganizationMember( + id=uuid4(), + organization_id=organization.id, + user_id=uuid4(), + role="owner", + ) + app = _build_test_app( + session_maker, + OrganizationContext(organization=organization, member=member), + ) + + async def _fake_fetch_provider_usage(**kwargs: object) -> ProviderUsageLive: + result = ProviderUsageLive( + provider=str(kwargs["provider"]), + account_key=str(kwargs["account_key"]), + checked_at=utcnow(), + reachable=True, + ) + result.models = ["claude-sonnet-4-6"] + result.raw_headers = { + "anthropic-ratelimit-tokens-limit": "100000", + } + return result + + monkeypatch.setattr( + "app.api.provider_credentials.fetch_provider_usage", + _fake_fetch_provider_usage, + ) + + try: + async with session_maker() as session: + session.add(organization) + await session.commit() + + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://testserver", + ) as client: + response = await client.post( + "/api/v1/provider-credentials/test", + json={ + "provider": "anthropic", + "account_key": "Claude", + "api_key": "sk-ant-test", + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["provider"] == "anthropic" + assert data["account_key"] == "Claude" + assert data["reachable"] is True + assert data["models"] == ["claude-sonnet-4-6"] + assert data["debug_rate_limit_headers"] == ["anthropic-ratelimit-tokens-limit"] + finally: + await engine.dispose() diff --git a/frontend/src/app/settings/ai-providers/page.tsx b/frontend/src/app/settings/ai-providers/page.tsx index 159ff45..e228c15 100644 --- a/frontend/src/app/settings/ai-providers/page.tsx +++ b/frontend/src/app/settings/ai-providers/page.tsx @@ -12,6 +12,7 @@ import { Input } from "@/components/ui/input"; import { ConfirmActionDialog } from "@/components/ui/confirm-action-dialog"; import { useOrganizationMembership } from "@/lib/use-organization-membership"; import type { ProviderCredentialRead, ProviderUsageLiveRead } from "@/api/generated/model"; +import { customFetch } from "@/api/mutator"; import { listProviderCredentialsApiV1ProviderCredentialsGet, createProviderCredentialApiV1ProviderCredentialsPost, @@ -79,6 +80,11 @@ interface CredentialFormProps { api_key: string; base_url: string; }) => Promise; + onTest: (data: { + account_key: string; + api_key: string; + base_url: string; + }) => Promise; onCancel: () => void; isSaving: boolean; error: string | null; @@ -91,6 +97,7 @@ function CredentialForm({ keyPlaceholder, accountKeyDefault, onSave, + onTest, onCancel, isSaving, error, @@ -99,6 +106,28 @@ function CredentialForm({ const [displayName, setDisplayName] = useState(""); const [apiKey, setApiKey] = useState(""); const [baseUrl, setBaseUrl] = useState(""); + const [isTesting, setIsTesting] = useState(false); + const [testResult, setTestResult] = useState(null); + const [testError, setTestError] = useState(null); + const canTest = showBaseUrl ? Boolean(baseUrl.trim()) : Boolean(apiKey.trim()); + + const runTest = async () => { + setIsTesting(true); + setTestResult(null); + setTestError(null); + try { + const result = await onTest({ + account_key: accountKey.trim() || accountKeyDefault || "test", + api_key: apiKey.trim(), + base_url: baseUrl.trim(), + }); + setTestResult(result); + } catch (err) { + setTestError(err instanceof Error ? err.message : "Test failed."); + } finally { + setIsTesting(false); + } + }; return (
@@ -160,6 +189,19 @@ function CredentialForm({

{error}

)}
+
+ {testError && ( +

{testError}

+ )} + {testResult && ( +
+

+ {testResult.reachable ? "Connection successful" : "Connection failed"} +

+

+ {testResult.error ?? `${testResult.models?.length ?? 0} model${(testResult.models?.length ?? 0) === 1 ? "" : "s"} returned`} +

+ {testResult.debug_rate_limit_headers && testResult.debug_rate_limit_headers.length > 0 && ( +

+ Rate-limit headers: {testResult.debug_rate_limit_headers.join(", ")} +

+ )} +
+ )}
); @@ -460,11 +526,12 @@ interface ProviderSectionProps { credentials: ProviderCredentialRead[]; isAdmin: boolean; onAdd: (providerId: ProviderId, data: { account_key: string; display_name: string; api_key: string; base_url: string }) => Promise; + onTest: (providerId: ProviderId, data: { account_key: string; api_key: string; base_url: string }) => Promise; onDelete: (cred: ProviderCredentialRead) => void; onToggle: (cred: ProviderCredentialRead) => Promise; } -function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onToggle }: ProviderSectionProps) { +function ProviderSection({ provider, credentials, isAdmin, onAdd, onTest, onDelete, onToggle }: ProviderSectionProps) { const Icon = provider.icon; const [showForm, setShowForm] = useState(false); const [saving, setSaving] = useState(false); @@ -516,6 +583,7 @@ function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onTo setSaving(false); } }} + onTest={(data) => onTest(provider.id, data)} onCancel={() => setShowForm(false)} isSaving={saving} error={saveError} @@ -595,6 +663,26 @@ export default function AIProvidersSettingsPage() { } }; + const handleTest = async ( + providerId: ProviderId, + data: { account_key: string; api_key: string; base_url: string }, + ): Promise => { + const response = await customFetch<{ + data: ProviderUsageLiveRead; + status: number; + headers: Headers; + }>("/api/v1/provider-credentials/test", { + method: "POST", + body: JSON.stringify({ + provider: providerId, + account_key: data.account_key || "test", + api_key: data.api_key || undefined, + base_url: data.base_url || undefined, + }), + }); + return response.data; + }; + const handleToggle = async (cred: ProviderCredentialRead) => { const res = await updateProviderCredentialApiV1ProviderCredentialsCredentialIdPatch( cred.id, @@ -654,6 +742,7 @@ export default function AIProvidersSettingsPage() { credentials={credentials.filter((c) => c.provider === provider.id)} isAdmin={isAdmin} onAdd={handleAdd} + onTest={handleTest} onDelete={setDeleteTarget} onToggle={handleToggle} />