null
/
Pipeline
1
0
Fork 0
Code Issues 15 Pull Requests Packages Projects Releases Wiki Activity

feat(scripts): credentials usage

This commit is contained in:
null 2026-05-20 23:03:19 -05:00
parent 07b47ace8f
commit ebba838025
4 changed files with 344 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
backend/app/api/provider_credentials.py
View File

@ -21,6 +21,7 @@ from app.models.provider_credentials import ProviderCredential, SUPPORTED_PROVID
from app.schemas.provider_credentials import (
ProviderCredentialCreate,
ProviderCredentialRead,
ProviderCredentialTestRequest,
ProviderCredentialUpdate,
ProviderUsageLiveRead,
RequestWindowRead,
@ -116,6 +117,60 @@ async def create_provider_credential(
return _to_read(cred)
@router.post("/test", response_model=ProviderUsageLiveRead)
async def test_provider_credential(
payload: ProviderCredentialTestRequest,
ctx: OrganizationContext = ORG_ADMIN_DEP,
) -> ProviderUsageLiveRead:
"""Validate provider credentials without saving them. Admin-only."""
if payload.provider not in SUPPORTED_PROVIDERS:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Unsupported provider '{payload.provider}'. Supported: {sorted(SUPPORTED_PROVIDERS)}",
)
account_key = payload.account_key.strip() or "test"
live = await fetch_provider_usage(
credential_id=f"test:{ctx.organization.id}:{payload.provider}:{account_key}",
provider=payload.provider,
account_key=account_key,
api_key=payload.api_key,
base_url=payload.base_url,
force_refresh=True,
)
def _tok(w) -> TokenWindowRead:
return TokenWindowRead(
limit=w.limit,
remaining=w.remaining,
used=w.used,
pct_used=w.pct_used,
reset_at=w.reset_at.isoformat() if w.reset_at else None,
reset_in_ms=w.reset_in_ms,
)
def _req(w) -> RequestWindowRead:
return RequestWindowRead(
limit=w.limit,
remaining=w.remaining,
reset_at=w.reset_at.isoformat() if w.reset_at else None,
reset_in_ms=w.reset_in_ms,
)
return ProviderUsageLiveRead(
provider=live.provider,
account_key=live.account_key,
checked_at=live.checked_at.isoformat(),
reachable=live.reachable,
error=live.error,
tokens=_tok(live.tokens),
input_tokens=_tok(live.input_tokens),
requests=_req(live.requests),
models=live.models,
debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None,
)
@router.get("/{credential_id}", response_model=ProviderCredentialRead)
async def get_provider_credential(
credential_id: UUID,
@ -210,6 +265,7 @@ async def get_provider_usage_live(
input_tokens=_tok(live.input_tokens),
requests=_req(live.requests),
models=live.models,
debug_rate_limit_headers=sorted(live.raw_headers.keys()) if live.raw_headers else None,
)

9
backend/app/schemas/provider_credentials.py
View File

@ -19,6 +19,13 @@ class ProviderCredentialCreate(SQLModel):
active: bool = True
class ProviderCredentialTestRequest(SQLModel):
provider: str
account_key: str = "test"
api_key: str | None = None
base_url: str | None = None
class ProviderCredentialUpdate(SQLModel):
display_name: str | None = None
api_key: str | None = None # None = keep existing; "" = clear it
@ -54,6 +61,8 @@ class ProviderUsageLiveRead(SQLModel):
input_tokens: TokenWindowRead # Anthropic splits input tokens separately
requests: RequestWindowRead
models: list[str] = []
# Optional debugging aid: exact rate-limit header names returned by provider.
debug_rate_limit_headers: list[str] | None = None
class ProviderCredentialRead(SQLModel):

189
backend/tests/test_provider_credentials_usage_api.py Normal file
View File

@ -0,0 +1,189 @@
# ruff: noqa: INP001
"""Integration tests for provider credential live-usage API behavior."""
from __future__ import annotations
from uuid import uuid4
import pytest
from fastapi import APIRouter, FastAPI
from httpx import ASGITransport, AsyncClient
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession
from app.api.deps import require_org_admin, require_org_member
from app.api.provider_credentials import router as provider_credentials_router
from app.core.time import utcnow
from app.db.session import get_session
from app.models.organization_members import OrganizationMember
from app.models.organizations import Organization
from app.models.provider_credentials import ProviderCredential
from app.services.organizations import OrganizationContext
from app.services.provider_usage import ProviderUsageLive
async def _make_engine() -> AsyncEngine:
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
return engine
def _build_test_app(
session_maker: async_sessionmaker[AsyncSession],
ctx: OrganizationContext,
) -> FastAPI:
app = FastAPI()
api_v1 = APIRouter(prefix="/api/v1")
api_v1.include_router(provider_credentials_router)
app.include_router(api_v1)
async def _override_get_session() -> AsyncSession:
async with session_maker() as session:
yield session
async def _override_require_org_member() -> OrganizationContext:
return ctx
async def _override_require_org_admin() -> OrganizationContext:
return ctx
app.dependency_overrides[get_session] = _override_get_session
app.dependency_overrides[require_org_member] = _override_require_org_member
app.dependency_overrides[require_org_admin] = _override_require_org_admin
return app
@pytest.mark.asyncio
async def test_usage_response_includes_rate_limit_header_names(monkeypatch: pytest.MonkeyPatch) -> None:
engine = await _make_engine()
session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
organization = Organization(id=uuid4(), name="Pipeline")
member = OrganizationMember(
id=uuid4(),
organization_id=organization.id,
user_id=uuid4(),
role="owner",
)
app = _build_test_app(
session_maker,
OrganizationContext(organization=organization, member=member),
)
credential = ProviderCredential(
id=uuid4(),
organization_id=organization.id,
provider="anthropic",
account_key="Claude",
display_name="Claude",
api_key="sk-ant-test",
api_key_last_four="test",
active=True,
)
async def _fake_fetch_provider_usage(**_: object) -> ProviderUsageLive:
result = ProviderUsageLive(
provider="anthropic",
account_key="Claude",
checked_at=utcnow(),
reachable=True,
)
result.raw_headers = {
"anthropic-ratelimit-requests-limit": "1000",
"anthropic-ratelimit-requests-remaining": "999",
"anthropic-ratelimit-requests-reset": "2026-05-21T12:00:00Z",
}
return result
monkeypatch.setattr(
"app.api.provider_credentials.fetch_provider_usage",
_fake_fetch_provider_usage,
)
try:
async with session_maker() as session:
session.add(organization)
session.add(credential)
await session.commit()
async with AsyncClient(
transport=ASGITransport(app=app),
base_url="http://testserver",
) as client:
response = await client.get(f"/api/v1/provider-credentials/{credential.id}/usage")
assert response.status_code == 200
data = response.json()
assert data["provider"] == "anthropic"
assert data["reachable"] is True
assert data["debug_rate_limit_headers"] == [
"anthropic-ratelimit-requests-limit",
"anthropic-ratelimit-requests-remaining",
"anthropic-ratelimit-requests-reset",
]
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_test_endpoint_returns_live_result(monkeypatch: pytest.MonkeyPatch) -> None:
engine = await _make_engine()
session_maker = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
organization = Organization(id=uuid4(), name="Pipeline")
member = OrganizationMember(
id=uuid4(),
organization_id=organization.id,
user_id=uuid4(),
role="owner",
)
app = _build_test_app(
session_maker,
OrganizationContext(organization=organization, member=member),
)
async def _fake_fetch_provider_usage(**kwargs: object) -> ProviderUsageLive:
result = ProviderUsageLive(
provider=str(kwargs["provider"]),
account_key=str(kwargs["account_key"]),
checked_at=utcnow(),
reachable=True,
)
result.models = ["claude-sonnet-4-6"]
result.raw_headers = {
"anthropic-ratelimit-tokens-limit": "100000",
}
return result
monkeypatch.setattr(
"app.api.provider_credentials.fetch_provider_usage",
_fake_fetch_provider_usage,
)
try:
async with session_maker() as session:
session.add(organization)
await session.commit()
async with AsyncClient(
transport=ASGITransport(app=app),
base_url="http://testserver",
) as client:
response = await client.post(
"/api/v1/provider-credentials/test",
json={
"provider": "anthropic",
"account_key": "Claude",
"api_key": "sk-ant-test",
},
)
assert response.status_code == 200
data = response.json()
assert data["provider"] == "anthropic"
assert data["account_key"] == "Claude"
assert data["reachable"] is True
assert data["models"] == ["claude-sonnet-4-6"]
assert data["debug_rate_limit_headers"] == ["anthropic-ratelimit-tokens-limit"]
finally:
await engine.dispose()

91
frontend/src/app/settings/ai-providers/page.tsx
View File

@ -12,6 +12,7 @@ import { Input } from "@/components/ui/input";
import { ConfirmActionDialog } from "@/components/ui/confirm-action-dialog";
import { useOrganizationMembership } from "@/lib/use-organization-membership";
import type { ProviderCredentialRead, ProviderUsageLiveRead } from "@/api/generated/model";
import { customFetch } from "@/api/mutator";
import {
listProviderCredentialsApiV1ProviderCredentialsGet,
createProviderCredentialApiV1ProviderCredentialsPost,
@ -79,6 +80,11 @@ interface CredentialFormProps {
api_key: string;
base_url: string;
}) => Promise<void>;
onTest: (data: {
account_key: string;
api_key: string;
base_url: string;
}) => Promise<ProviderUsageLiveRead>;
onCancel: () => void;
isSaving: boolean;
error: string | null;
@ -91,6 +97,7 @@ function CredentialForm({
keyPlaceholder,
accountKeyDefault,
onSave,
onTest,
onCancel,
isSaving,
error,
@ -99,6 +106,28 @@ function CredentialForm({
const [displayName, setDisplayName] = useState("");
const [apiKey, setApiKey] = useState("");
const [baseUrl, setBaseUrl] = useState("");
const [isTesting, setIsTesting] = useState(false);
const [testResult, setTestResult] = useState<ProviderUsageLiveRead | null>(null);
const [testError, setTestError] = useState<string | null>(null);
const canTest = showBaseUrl ? Boolean(baseUrl.trim()) : Boolean(apiKey.trim());
const runTest = async () => {
setIsTesting(true);
setTestResult(null);
setTestError(null);
try {
const result = await onTest({
account_key: accountKey.trim() || accountKeyDefault || "test",
api_key: apiKey.trim(),
base_url: baseUrl.trim(),
});
setTestResult(result);
} catch (err) {
setTestError(err instanceof Error ? err.message : "Test failed.");
} finally {
setIsTesting(false);
}
};
return (
<div className="mt-3 rounded-xl border border-[color:var(--border)] bg-[color:var(--surface-muted)] p-4">
@ -160,6 +189,19 @@ function CredentialForm({
<p className="text-sm text-[color:var(--danger)]">{error}</p>
)}
<div className="flex gap-2">
<Button
variant="outline"
size="sm"
disabled={isSaving || isTesting || !canTest}
onClick={runTest}
>
{isTesting ? (
<Loader2 className="mr-1.5 h-3.5 w-3.5 animate-spin" />
) : (
<RefreshCw className="mr-1.5 h-3.5 w-3.5" />
)}
{isTesting ? "Testing…" : "Test"}
</Button>
<Button
size="sm"
disabled={isSaving || !accountKey.trim() || (showBaseUrl && !baseUrl.trim())}
@ -179,6 +221,30 @@ function CredentialForm({
Cancel
</Button>
</div>
{testError && (
<p className="text-sm text-[color:var(--danger)]">{testError}</p>
)}
{testResult && (
<div className={`rounded-lg border p-3 text-xs ${
testResult.reachable
? "border-[color:rgba(52,211,153,0.35)] bg-[color:rgba(52,211,153,0.08)]"
: "border-[color:rgba(248,113,113,0.35)] bg-[color:rgba(248,113,113,0.08)]"
}`}>
<p className={`font-medium ${
testResult.reachable ? "text-[color:var(--success)]" : "text-[color:var(--danger)]"
}`}>
{testResult.reachable ? "Connection successful" : "Connection failed"}
</p>
<p className="mt-1 text-muted">
{testResult.error ?? `${testResult.models?.length ?? 0} model${(testResult.models?.length ?? 0) === 1 ? "" : "s"} returned`}
</p>
{testResult.debug_rate_limit_headers && testResult.debug_rate_limit_headers.length > 0 && (
<p className="mt-1 text-muted">
Rate-limit headers: {testResult.debug_rate_limit_headers.join(", ")}
</p>
)}
</div>
)}
</div>
</div>
);
@ -460,11 +526,12 @@ interface ProviderSectionProps {
credentials: ProviderCredentialRead[];
isAdmin: boolean;
onAdd: (providerId: ProviderId, data: { account_key: string; display_name: string; api_key: string; base_url: string }) => Promise<void>;
onTest: (providerId: ProviderId, data: { account_key: string; api_key: string; base_url: string }) => Promise<ProviderUsageLiveRead>;
onDelete: (cred: ProviderCredentialRead) => void;
onToggle: (cred: ProviderCredentialRead) => Promise<void>;
}
function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onToggle }: ProviderSectionProps) {
function ProviderSection({ provider, credentials, isAdmin, onAdd, onTest, onDelete, onToggle }: ProviderSectionProps) {
const Icon = provider.icon;
const [showForm, setShowForm] = useState(false);
const [saving, setSaving] = useState(false);
@ -516,6 +583,7 @@ function ProviderSection({ provider, credentials, isAdmin, onAdd, onDelete, onTo
setSaving(false);
}
}}
onTest={(data) => onTest(provider.id, data)}
onCancel={() => setShowForm(false)}
isSaving={saving}
error={saveError}
@ -595,6 +663,26 @@ export default function AIProvidersSettingsPage() {
}
};
const handleTest = async (
providerId: ProviderId,
data: { account_key: string; api_key: string; base_url: string },
): Promise<ProviderUsageLiveRead> => {
const response = await customFetch<{
data: ProviderUsageLiveRead;
status: number;
headers: Headers;
}>("/api/v1/provider-credentials/test", {
method: "POST",
body: JSON.stringify({
provider: providerId,
account_key: data.account_key || "test",
api_key: data.api_key || undefined,
base_url: data.base_url || undefined,
}),
});
return response.data;
};
const handleToggle = async (cred: ProviderCredentialRead) => {
const res = await updateProviderCredentialApiV1ProviderCredentialsCredentialIdPatch(
cred.id,
@ -654,6 +742,7 @@ export default function AIProvidersSettingsPage() {
credentials={credentials.filter((c) => c.provider === provider.id)}
isAdmin={isAdmin}
onAdd={handleAdd}
onTest={handleTest}
onDelete={setDeleteTarget}
onToggle={handleToggle}
/>