feat(scripts): credentials usage

backend/app/api/provider_credentials.py

@@ -21,6 +21,7 @@ from app.models.provider_credentials import ProviderCredential, SUPPORTED_PROVID
 from app.schemas.provider_credentials import (
     ProviderCredentialCreate,
     ProviderCredentialRead,
+    ProviderCredentialTestRequest,
     ProviderCredentialUpdate,
     ProviderUsageLiveRead,
     RequestWindowRead,
@@ -116,6 +117,60 @@ async def create_provider_credential(
     return _to_read(cred)
 
 
+@router.post("/test", response_model=ProviderUsageLiveRead)
+async def test_provider_credential(
+    payload: ProviderCredentialTestRequest,
+    ctx: OrganizationContext = ORG_ADMIN_DEP,
+) -> ProviderUsageLiveRead:
+    """Validate provider credentials without saving them. Admin-only."""
+    if payload.provider not in SUPPORTED_PROVIDERS:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Unsupported provider '{payload.provider}'. Supported: {sorted(SUPPORTED_PROVIDERS)}",
+        )
+
+    account_key = payload.account_key.strip() or "test"
+    live = await fetch_provider_usage(
+        credential_id=f"test:{ctx.organization.id}:{payload.provider}:{account_key}",
+        provider=payload.provider,
+        account_key=account_key,
+        api_key=payload.api_key,
+        base_url=payload.base_url,
+        force_refresh=True,
+    )
+
+    def _tok(w) -> TokenWindowRead:
+        return TokenWindowRead(
+            limit=w.limit,
+            remaining=w.remaining,
+            used=w.used,
+            pct_used=w.pct_used,
+            reset_at=w.reset_at.isoformat() if w.reset_at else None,
+            reset_in_ms=w.reset_in_ms,
+        )
+
+    def _req(w) -> RequestWindowRead:
+        return RequestWindowRead(
+            limit=w.limit,
+            remaining=w.remaining,
+            reset_at=w.reset_at.isoformat() if w.reset_at else None,
+            reset_in_ms=w.reset_in_ms,
+        )
+
+    return ProviderUsageLiveRead(
+        provider=live.provider,
+        account_key=live.account_key,
+        checked_at=live.checked_at.isoformat(),
+        reachable=live.reachable,
+        error=live.error,
+        tokens=_tok(live.tokens),
+        input_tokens=_tok(live.input_tokens),
+        requests=_req(live.requests),
+        models=live.models,
+        debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None,
+    )
+
+
 @router.get("/{credential_id}", response_model=ProviderCredentialRead)
 async def get_provider_credential(
     credential_id: UUID,
@@ -210,6 +265,7 @@ async def get_provider_usage_live(
         input_tokens=_tok(live.input_tokens),
         requests=_req(live.requests),
         models=live.models,
+        debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None,
     )
 
 

backend/app/schemas/provider_credentials.py

@@ -19,6 +19,13 @@ class ProviderCredentialCreate(SQLModel):
     active: bool = True
 
 
+class ProviderCredentialTestRequest(SQLModel):
+    provider: str
+    account_key: str = "test"
+    api_key: str | None = None
+    base_url: str | None = None
+
+
 class ProviderCredentialUpdate(SQLModel):
     display_name: str | None = None
     api_key: str | None = None       # None = keep existing; "" = clear it
@@ -54,6 +61,8 @@ class ProviderUsageLiveRead(SQLModel):
     input_tokens: TokenWindowRead   # Anthropic splits input tokens separately
     requests: RequestWindowRead
     models: list[str] = []
+    # Optional debugging aid: exact rate-limit header names returned by provider.
+    debug_rate_limit_headers: list[str] | None = None
 
 
 class ProviderCredentialRead(SQLModel):

backend/tests/test_provider_credentials_usage_api.py

@@ -0,0 +1,189 @@
+# ruff: noqa: INP001
+"""Integration tests for provider credential live-usage API behavior."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+import pytest
+from fastapi import APIRouter, FastAPI
+from httpx import ASGITransport, AsyncClient
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
+from sqlmodel import SQLModel
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from app.api.deps import require_org_admin, require_org_member
+from app.api.provider_credentials import router as provider_credentials_router
+from app.core.time import utcnow
+from app.db.session import get_session
+from app.models.organization_members import OrganizationMember
+from app.models.organizations import Organization
+from app.models.provider_credentials import ProviderCredential
+from app.services.organizations import OrganizationContext
+from app.services.provider_usage import ProviderUsageLive
+
+
+async def _make_engine() -> AsyncEngine:
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    async with engine.begin() as conn:
+        await conn.run_sync(SQLModel.metadata.create_all)
+    return engine
+
+
+def _build_test_app(
+    session_maker: async_sessionmaker[AsyncSession],
+    ctx: OrganizationContext,
+) -> FastAPI:
+    app = FastAPI()
+    api_v1 = APIRouter(prefix="/api/v1")
+    api_v1.include_router(provider_credentials_router)
+    app.include_router(api_v1)
+
+    async def _override_get_session() -> AsyncSession:
+        async with session_maker() as session:
+            yield session
+
+    async def _override_require_org_member() -> OrganizationContext:
+        return ctx
+
+    async def _override_require_org_admin() -> OrganizationContext:
+        return ctx
+
+    app.dependency_overrides[get_session] = _override_get_session
+    app.dependency_overrides[require_org_member] = _override_require_org_member
+    app.dependency_overrides[require_org_admin] = _override_require_org_admin
+    return app
+
+
+@pytest.mark.asyncio
+async def test_usage_response_includes_rate_limit_header_names(monkeypatch: pytest.MonkeyPatch) -> None:
+    engine = await _make_engine()
+    session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    organization = Organization(id=uuid4(), name="Pipeline")
+    member = OrganizationMember(
+        id=uuid4(),
+        organization_id=organization.id,
+        user_id=uuid4(),
+        role="owner",
+    )
+    app = _build_test_app(
+        session_maker,
+        OrganizationContext(organization=organization, member=member),
+    )
+
+    credential = ProviderCredential(
+        id=uuid4(),
+        organization_id=organization.id,
+        provider="anthropic",
+        account_key="Claude",
+        display_name="Claude",
+        api_key="sk-ant-test",
+        api_key_last_four="test",
+        active=True,
+    )
+
+    async def _fake_fetch_provider_usage(**_: object) -> ProviderUsageLive:
+        result = ProviderUsageLive(
+            provider="anthropic",
+            account_key="Claude",
+            checked_at=utcnow(),
+            reachable=True,
+        )
+        result.raw_headers = {
+            "anthropic-ratelimit-requests-limit": "1000",
+            "anthropic-ratelimit-requests-remaining": "999",
+            "anthropic-ratelimit-requests-reset": "2026-05-21T12:00:00Z",
+        }
+        return result
+
+    monkeypatch.setattr(
+        "app.api.provider_credentials.fetch_provider_usage",
+        _fake_fetch_provider_usage,
+    )
+
+    try:
+        async with session_maker() as session:
+            session.add(organization)
+            session.add(credential)
+            await session.commit()
+
+        async with AsyncClient(
+            transport=ASGITransport(app=app),
+            base_url="http://testserver",
+        ) as client:
+            response = await client.get(f"/api/v1/provider-credentials/{credential.id}/usage")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["provider"] == "anthropic"
+        assert data["reachable"] is True
+        assert data["debug_rate_limit_headers"] == [
+            "anthropic-ratelimit-requests-limit",
+            "anthropic-ratelimit-requests-remaining",
+            "anthropic-ratelimit-requests-reset",
+        ]
+    finally:
+        await engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_test_endpoint_returns_live_result(monkeypatch: pytest.MonkeyPatch) -> None:
+    engine = await _make_engine()
+    session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    organization = Organization(id=uuid4(), name="Pipeline")
+    member = OrganizationMember(
+        id=uuid4(),
+        organization_id=organization.id,
+        user_id=uuid4(),
+        role="owner",
+    )
+    app = _build_test_app(
+        session_maker,
+        OrganizationContext(organization=organization, member=member),
+    )
+
+    async def _fake_fetch_provider_usage(**kwargs: object) -> ProviderUsageLive:
+        result = ProviderUsageLive(
+            provider=str(kwargs["provider"]),
+            account_key=str(kwargs["account_key"]),
+            checked_at=utcnow(),
+            reachable=True,
+        )
+        result.models = ["claude-sonnet-4-6"]
+        result.raw_headers = {
+            "anthropic-ratelimit-tokens-limit": "100000",
+        }
+        return result
+
+    monkeypatch.setattr(
+        "app.api.provider_credentials.fetch_provider_usage",
+        _fake_fetch_provider_usage,
+    )
+
+    try:
+        async with session_maker() as session:
+            session.add(organization)
+            await session.commit()
+
+        async with AsyncClient(
+            transport=ASGITransport(app=app),
+            base_url="http://testserver",
+        ) as client:
+            response = await client.post(
+                "/api/v1/provider-credentials/test",
+                json={
+                    "provider": "anthropic",
+                    "account_key": "Claude",
+                    "api_key": "sk-ant-test",
+                },
+            )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["provider"] == "anthropic"
+        assert data["account_key"] == "Claude"
+        assert data["reachable"] is True
+        assert data["models"] == ["claude-sonnet-4-6"]
+        assert data["debug_rate_limit_headers"] == ["anthropic-ratelimit-tokens-limit"]
+    finally:
+        await engine.dispose()

frontend/src/app/settings/ai-providers/page.tsx

@@ -12,6 +12,7 @@ import { Input } from "@/components/ui/input";
 import { ConfirmActionDialog } from "@/components/ui/confirm-action-dialog";
 import { useOrganizationMembership } from "@/lib/use-organization-membership";
 import type { ProviderCredentialRead, ProviderUsageLiveRead } from "@/api/generated/model";
+import { customFetch } from "@/api/mutator";
 import {
   listProviderCredentialsApiV1ProviderCredentialsGet,
   createProviderCredentialApiV1ProviderCredentialsPost,
@@ -79,6 +80,11 @@ interface CredentialFormProps {
     api_key: string;
     base_url: string;
   }) => Promise<void>;
+  onTest: (data: {
+    account_key: string;
+    api_key: string;
+    base_url: string;
+  }) => Promise<ProviderUsageLiveRead>;
   onCancel: () => void;
   isSaving: boolean;
   error: string | null;
@@ -91,6 +97,7 @@ function CredentialForm({
   keyPlaceholder,
   accountKeyDefault,
   onSave,
+  onTest,
   onCancel,
   isSaving,
   error,
@@ -99,6 +106,28 @@ function CredentialForm({
   const [displayName, setDisplayName] = useState("");
   const [apiKey, setApiKey] = useState("");
   const [baseUrl, setBaseUrl] = useState("");
+  const [isTesting, setIsTesting] = useState(false);
+  const [testResult, setTestResult] = useState<ProviderUsageLiveRead | null>(null);
+  const [testError, setTestError] = useState<string | null>(null);
+  const canTest = showBaseUrl ? Boolean(baseUrl.trim()) : Boolean(apiKey.trim());
+
+  const runTest = async () => {
+    setIsTesting(true);
+    setTestResult(null);
+    setTestError(null);
+    try {
+      const result = await onTest({
+        account_key: accountKey.trim() || accountKeyDefault || "test",
+        api_key: apiKey.trim(),
+        base_url: baseUrl.trim(),
+      });
+      setTestResult(result);
+    } catch (err) {
+      setTestError(err instanceof Error ? err.message : "Test failed.");
+    } finally {
+      setIsTesting(false);
+    }
+  };
 
   return (
     <div className="mt-3 rounded-xl border border-[color:var(--border)] bg-[color:var(--surface-muted)] p-4">
@@ -160,6 +189,19 @@ function CredentialForm({
           <p className="text-sm text-[color:var(--danger)]">{error}</p>
         )}
         <div className="flex gap-2">
+          <Button
+            variant="outline"
+            size="sm"
+            disabled={isSaving || isTesting || !canTest}
+            onClick={runTest}
+          >
+            {isTesting ? (
+              <Loader2 className="mr-1.5 h-3.5 w-3.5 animate-spin" />
+            ) : (
+              <RefreshCw className="mr-1.5 h-3.5 w-3.5" />
+            )}
+            {isTesting ? "Testing…" : "Test"}
+          </Button>
           <Button
             size="sm"
             disabled={isSaving || !accountKey.trim() || (showBaseUrl && !baseUrl.trim())}
@@ -179,6 +221,30 @@ function CredentialForm({
             Cancel
           </Button>
         </div>
+        {testError && (
+          <p className="text-sm text-[color:var(--danger)]">{testError}</p>
+        )}
+        {testResult && (
+          <div className={`rounded-lg border p-3 text-xs ${
+            testResult.reachable
+              ? "border-[color:rgba(52,211,153,0.35)] bg-[color:rgba(52,211,153,0.08)]"
+              : "border-[color:rgba(248,113,113,0.35)] bg-[color:rgba(248,113,113,0.08)]"
+          }`}>
+            <p className={`font-medium ${
+              testResult.reachable ? "text-[color:var(--success)]" : "text-[color:var(--danger)]"
+            }`}>
+              {testResult.reachable ? "Connection successful" : "Connection failed"}
+            </p>
+            <p className="mt-1 text-muted">
+              {testResult.error ?? `${testResult.models?.length ?? 0} model${(testResult.models?.length ?? 0) === 1 ? "" : "s"} returned`}
+            </p>
+            {testResult.debug_rate_limit_headers && testResult.debug_rate_limit_headers.length > 0 && (
+              <p className="mt-1 text-muted">
+                Rate-limit headers: {testResult.debug_rate_limit_headers.join(", ")}
+              </p>
+            )}
+          </div>
+        )}
       </div>
     </div>
   );
@@ -460,11 +526,12 @@ interface ProviderSectionProps {
   credentials: ProviderCredentialRead[];
   isAdmin: boolean;
   onAdd: (providerId: ProviderId, data: { account_key: string; display_name: string; api_key: string; base_url: string }) => Promise<void>;
+  onTest: (providerId: ProviderId, data: { account_key: string; api_key: string; base_url: string }) => Promise<ProviderUsageLiveRead>;
   onDelete: (cred: ProviderCredentialRead) => void;
   onToggle: (cred: ProviderCredentialRead) => Promise<void>;
 }
 
-function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onToggle }: ProviderSectionProps) {
+function ProviderSection({ provider, credentials, isAdmin, onAdd, onTest, onDelete, onToggle }: ProviderSectionProps) {
   const Icon = provider.icon;
   const [showForm, setShowForm] = useState(false);
   const [saving, setSaving] = useState(false);
@@ -516,6 +583,7 @@ function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onTo
               setSaving(false);
             }
           }}
+          onTest={(data) => onTest(provider.id, data)}
           onCancel={() => setShowForm(false)}
           isSaving={saving}
           error={saveError}
@@ -595,6 +663,26 @@ export default function AIProvidersSettingsPage() {
     }
   };
 
+  const handleTest = async (
+    providerId: ProviderId,
+    data: { account_key: string; api_key: string; base_url: string },
+  ): Promise<ProviderUsageLiveRead> => {
+    const response = await customFetch<{
+      data: ProviderUsageLiveRead;
+      status: number;
+      headers: Headers;
+    }>("/api/v1/provider-credentials/test", {
+      method: "POST",
+      body: JSON.stringify({
+        provider: providerId,
+        account_key: data.account_key || "test",
+        api_key: data.api_key || undefined,
+        base_url: data.base_url || undefined,
+      }),
+    });
+    return response.data;
+  };
+
   const handleToggle = async (cred: ProviderCredentialRead) => {
     const res = await updateProviderCredentialApiV1ProviderCredentialsCredentialIdPatch(
       cred.id,
@@ -654,6 +742,7 @@ export default function AIProvidersSettingsPage() {
                 credentials={credentials.filter((c) => c.provider === provider.id)}
                 isAdmin={isAdmin}
                 onAdd={handleAdd}
+                onTest={handleTest}
                 onDelete={setDeleteTarget}
                 onToggle={handleToggle}
               />