diff --git a/.env.example b/.env.example index b6f1501..f44dbae 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,23 @@ AUTH_MODE=local # REQUIRED when AUTH_MODE=local (must be non-placeholder and at least 50 chars). LOCAL_AUTH_TOKEN= +# --- AI provider credentials (local mode only) --- +# Pipeline reads local credential files on every boot and upserts provider rows +# automatically — no manual configuration required if you have Claude Code or +# Codex CLI installed: +# +# Anthropic ~/.claude/.credentials.json (claudeAiOauth.accessToken) +# OpenAI ~/.codex/auth.json (tokens.access_token) +# +# Override the credential file paths if they live elsewhere: +# CLAUDE_CREDENTIALS_PATH=/path/to/.credentials.json +# CODEX_CREDENTIALS_PATH=/path/to/auth.json +# +# Supplement with explicit API keys (used alongside session tokens): +# ANTHROPIC_API_KEY=sk-ant-... +# OPENAI_API_KEY=sk-... +# OPENAI_BASE_URL=http://localhost:11434/v1 # Ollama, Azure, etc. + # --- frontend settings --- # REQUIRED: Public URL used by the browser to reach the API. # Use `auto` to target the same host currently serving Pipeline on port 8001. diff --git a/README.md b/README.md index dbc594f..c34e71d 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,62 @@ Set `AUTH_MODE=clerk` and configure `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` and `CLE - Backend: [`backend/.env.example`](./backend/.env.example) - Frontend: [`frontend/.env.example`](./frontend/.env.example) +## AI provider credentials + +When running in `AUTH_MODE=local`, Mission Control automatically configures AI provider credentials on startup by reading the credential files written by the Claude Code and Codex CLI VS Code extensions. + +### What gets read and where + +| Provider | File | Field | +|---|---|---| +| Anthropic / Claude | `~/.claude/.credentials.json` | `claudeAiOauth.accessToken` | +| OpenAI / GPT | `~/.codex/auth.json` | `tokens.access_token` | + +These are the same files the extensions update when you log in or out. +No manual configuration is required if either extension is installed and authenticated. + +### How it works + +On every boot the backend seeds `provider_credentials` rows from those files. +A background watcher then polls the file modification times every **60 seconds**. +When a change is detected (login, logout, token refresh) the affected row is updated automatically — you are reconfigured within one polling cycle of any credential change. + +Seeding is idempotent: rows are only written when the stored token differs from what is on disk. +If a file is absent or the token has expired, that provider is skipped without error. + +### Override credential file paths + +Set environment variables to point to non-default locations: + +```env +CLAUDE_CREDENTIALS_PATH=/path/to/.credentials.json +CODEX_CREDENTIALS_PATH=/path/to/auth.json +``` + +### Supplement with explicit API keys + +Session tokens from the local files grant subscription-level access. +To also configure standard API keys (for rate-limit or billing endpoints), add these to `.env`: + +```env +ANTHROPIC_API_KEY=sk-ant-... +OPENAI_API_KEY=sk-... +OPENAI_BASE_URL=http://localhost:11434/v1 # Ollama, Azure, or any OpenAI-compatible endpoint +``` + +### Run the seeder manually + +```bash +# one-shot seed +python backend/scripts/seed_provider_credentials.py + +# continuous watcher (re-seeds every 60 s when files change) +python backend/scripts/seed_provider_credentials.py --watch + +# custom poll interval +python backend/scripts/seed_provider_credentials.py --watch --interval 30 +``` + ## Documentation Complete guides for deployment, production, troubleshooting, and testing are in [`/docs`](./docs/). diff --git a/backend/app/main.py b/backend/app/main.py index d11fd52..3cf3f53 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any @@ -42,6 +43,7 @@ from app.api.users import router as users_router from app.core.config import settings from app.core.error_handling import install_error_handling from app.core.logging import configure_logging, get_logger +from app.core.auth_mode import AuthMode from app.core.rate_limit import validate_rate_limit_redis from app.core.rate_limit_backend import RateLimitBackend from app.core.security_headers import SecurityHeadersMiddleware @@ -471,10 +473,38 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: else: logger.info("app.lifecycle.rate_limit backend=memory") seed_scheduled_forgejo_sync() + if settings.auth_mode == AuthMode.LOCAL: + # Seed provider credentials from local credential files on every boot, + # then start a background watcher that re-seeds whenever a file changes + # (VS Code extension login/logout events). + credential_watcher_task = None + try: + from scripts.seed_provider_credentials import ( + seed as seed_providers, + watch as watch_providers, + ) + changed = await seed_providers(verbose=False) + if changed: + logger.info("app.lifecycle.provider_credentials seeded count=%d", changed) + else: + logger.debug("app.lifecycle.provider_credentials already current") + credential_watcher_task = asyncio.create_task( + watch_providers(), name="provider_credentials_watcher" + ) + except Exception as exc: + logger.warning("app.lifecycle.provider_credentials setup_failed error=%s", exc) logger.info("app.lifecycle.started") try: yield finally: + if settings.auth_mode == AuthMode.LOCAL: + task = locals().get("credential_watcher_task") + if task and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass logger.info("app.lifecycle.stopped") diff --git a/backend/scripts/seed_provider_credentials.py b/backend/scripts/seed_provider_credentials.py new file mode 100644 index 0000000..a76d607 --- /dev/null +++ b/backend/scripts/seed_provider_credentials.py @@ -0,0 +1,392 @@ +"""Idempotent seeder for local AI provider credentials. + +Reads session tokens from the local Claude Code and Codex CLI credential +files and upserts them as ProviderCredential rows so Pipeline recognises +the providers as configured. + +Sources +─────── + Anthropic / Claude + ~/.claude/.credentials.json (claudeAiOauth.accessToken) + Override path: CLAUDE_CREDENTIALS_PATH env var + + OpenAI / GPT + ~/.codex/auth.json (tokens.access_token) + Override path: CODEX_CREDENTIALS_PATH env var + + API keys (optional fallback / supplement) + ANTHROPIC_API_KEY + OPENAI_API_KEY + OPENAI_BASE_URL + +Safe to run on every boot — rows are only created or updated when a token +or key value differs from what is already stored. + +Usage +───── + # standalone + python scripts/seed_provider_credentials.py + + # called automatically from main.py lifespan when AUTH_MODE=local +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(BACKEND_ROOT)) + + +# ── credential-file readers ─────────────────────────────────────────────────── + +def _read_claude_session_key() -> str | None: + """Read the Claude Code OAuth access token from ~/.claude/.credentials.json.""" + path = os.environ.get("CLAUDE_CREDENTIALS_PATH", "").strip() + if not path: + path = os.path.join(os.path.expanduser("~"), ".claude", ".credentials.json") + try: + with open(path) as fh: + data = json.load(fh) + except (FileNotFoundError, PermissionError, ValueError, OSError): + return None + + oauth = data.get("claudeAiOauth") + if not isinstance(oauth, dict): + return None + + token = oauth.get("accessToken") + expires_at = oauth.get("expiresAt") + + if not isinstance(token, str) or not token: + return None + if isinstance(expires_at, (int, float)) and expires_at > 0: + if expires_at <= time.time() * 1000: + return None # expired + + return token + + +def _read_openai_session_key() -> str | None: + """Read the Codex CLI JWT access token from ~/.codex/auth.json.""" + path = os.environ.get("CODEX_CREDENTIALS_PATH", "").strip() + if not path: + path = os.path.join(os.path.expanduser("~"), ".codex", "auth.json") + try: + with open(path) as fh: + data = json.load(fh) + except (FileNotFoundError, PermissionError, ValueError, OSError): + return None + + tokens = data.get("tokens") + if not isinstance(tokens, dict): + return None + + token = tokens.get("access_token") + if not isinstance(token, str) or not token: + return None + + # Check JWT expiry without verifying the signature. + parts = token.split(".") + if len(parts) >= 2: + try: + pad = parts[1] + "=" * (4 - len(parts[1]) % 4) + payload = json.loads(base64.urlsafe_b64decode(pad)) + exp = payload.get("exp") + if isinstance(exp, (int, float)) and exp <= time.time(): + return None # expired + except Exception: + pass # Can't decode — proceed optimistically + + return token + + +# ── spec dataclass ──────────────────────────────────────────────────────────── + +@dataclass +class _ProviderSpec: + provider: str + account_key: str + display_name: str + api_key: str | None = None + session_key: str | None = None + base_url: str | None = None + + @property + def has_credentials(self) -> bool: + return bool(self.api_key or self.session_key or self.base_url) + + +def _collect_specs() -> list[_ProviderSpec]: + specs: list[_ProviderSpec] = [] + + # Anthropic: session token from local file, API key from env. + anthropic = _ProviderSpec( + provider="anthropic", + account_key="default", + display_name="Anthropic (local)", + session_key=_read_claude_session_key(), + api_key=os.environ.get("ANTHROPIC_API_KEY", "").strip() or None, + base_url=os.environ.get("ANTHROPIC_BASE_URL", "").strip() or None, + ) + if anthropic.has_credentials: + specs.append(anthropic) + + # OpenAI: session token from local Codex file, API key from env. + openai = _ProviderSpec( + provider="openai", + account_key="default", + display_name="OpenAI (local)", + session_key=_read_openai_session_key(), + api_key=os.environ.get("OPENAI_API_KEY", "").strip() or None, + base_url=os.environ.get("OPENAI_BASE_URL", "").strip() or None, + ) + if openai.has_credentials: + specs.append(openai) + + return specs + + +# ── upsert logic ────────────────────────────────────────────────────────────── + +async def seed(*, verbose: bool = True) -> int: + """Upsert provider credentials from local credential files and env vars. + + Returns the number of rows created or updated. + """ + from sqlmodel import select + + from app.core.auth import LOCAL_AUTH_USER_ID, LOCAL_AUTH_EMAIL, LOCAL_AUTH_NAME + from app.db import crud + from app.db.session import async_session_maker, init_db + from app.models.provider_credentials import ProviderCredential + from app.models.users import User + from app.services.organizations import ensure_member_for_user + + specs = _collect_specs() + if not specs: + if verbose: + print( + "seed_provider_credentials: no credentials found — " + "ensure ~/.claude/.credentials.json or ~/.codex/auth.json exist, " + "or set ANTHROPIC_API_KEY / OPENAI_API_KEY" + ) + return 0 + + await init_db() + + async with async_session_maker() as session: + # Ensure the local user + org exist before we can write credentials. + user, _created = await crud.get_or_create( + session, + User, + clerk_user_id=LOCAL_AUTH_USER_ID, + defaults={"email": LOCAL_AUTH_EMAIL, "name": LOCAL_AUTH_NAME}, + ) + if not user.email: + user.email = LOCAL_AUTH_EMAIL + if not user.name: + user.name = LOCAL_AUTH_NAME + session.add(user) + await session.commit() + await session.refresh(user) + + member = await ensure_member_for_user(session, user) + organization_id = member.organization_id + + changed = 0 + for spec in specs: + existing = ( + await session.exec( + select(ProviderCredential).where( + ProviderCredential.organization_id == organization_id, + ProviderCredential.provider == spec.provider, + ProviderCredential.account_key == spec.account_key, + ) + ) + ).first() + + if existing is None: + cred = ProviderCredential( + organization_id=organization_id, + provider=spec.provider, + account_key=spec.account_key, + display_name=spec.display_name, + api_key=spec.api_key, + api_key_last_four=spec.api_key[-4:] if spec.api_key else None, + session_key=spec.session_key, + session_key_last_four=spec.session_key[-4:] if spec.session_key else None, + base_url=spec.base_url, + active=True, + ) + session.add(cred) + await session.commit() + changed += 1 + if verbose: + _log_created(spec) + else: + dirty = False + if spec.api_key is not None and existing.api_key != spec.api_key: + existing.api_key = spec.api_key + existing.api_key_last_four = spec.api_key[-4:] + dirty = True + if spec.session_key is not None and existing.session_key != spec.session_key: + existing.session_key = spec.session_key + existing.session_key_last_four = spec.session_key[-4:] + dirty = True + if spec.base_url != existing.base_url: + existing.base_url = spec.base_url + dirty = True + if not existing.active: + existing.active = True + dirty = True + if dirty: + session.add(existing) + await session.commit() + changed += 1 + if verbose: + _log_updated(spec) + else: + if verbose: + print( + f"seed_provider_credentials: " + f"{spec.provider}/{spec.account_key} already current — skipped" + ) + + return changed + + +def _log_created(spec: _ProviderSpec) -> None: + hints: list[str] = [] + if spec.session_key: + hints.append(f"session …{spec.session_key[-4:]}") + if spec.api_key: + hints.append(f"api_key …{spec.api_key[-4:]}") + print( + f"seed_provider_credentials: created " + f"{spec.provider}/{spec.account_key} ({', '.join(hints) or 'base_url only'})" + ) + + +def _log_updated(spec: _ProviderSpec) -> None: + hints: list[str] = [] + if spec.session_key: + hints.append(f"session …{spec.session_key[-4:]}") + if spec.api_key: + hints.append(f"api_key …{spec.api_key[-4:]}") + print( + f"seed_provider_credentials: updated " + f"{spec.provider}/{spec.account_key} ({', '.join(hints) or 'base_url only'})" + ) + + +WATCH_INTERVAL_SECONDS = 60 # re-check credential files every 60 s + + +def _credential_file_paths() -> list[str]: + """Return the resolved paths for all watched credential files.""" + claude_path = os.environ.get("CLAUDE_CREDENTIALS_PATH", "").strip() + if not claude_path: + claude_path = os.path.join(os.path.expanduser("~"), ".claude", ".credentials.json") + codex_path = os.environ.get("CODEX_CREDENTIALS_PATH", "").strip() + if not codex_path: + codex_path = os.path.join(os.path.expanduser("~"), ".codex", "auth.json") + return [claude_path, codex_path] + + +def _snapshot_mtimes() -> dict[str, float]: + """Return {path: mtime} for each watched file, 0.0 if missing.""" + result: dict[str, float] = {} + for path in _credential_file_paths(): + try: + result[path] = os.stat(path).st_mtime + except OSError: + result[path] = 0.0 + return result + + +async def watch(*, interval: int = WATCH_INTERVAL_SECONDS) -> None: + """Background task: re-seed whenever a credential file changes. + + Polls file mtimes every `interval` seconds (default 60). Only re-seeds + when at least one file's mtime has changed — no DB or file-read overhead + on quiet cycles. + + Designed to be run as a long-lived asyncio task inside the FastAPI + lifespan. Exits cleanly on CancelledError. + """ + import logging + logger = logging.getLogger(__name__) + + last_mtimes = _snapshot_mtimes() + logger.info( + "provider_credentials.watcher started interval=%ds paths=%s", + interval, + list(last_mtimes.keys()), + ) + + try: + while True: + await asyncio.sleep(interval) + current = _snapshot_mtimes() + changed_paths = [p for p, mtime in current.items() if mtime != last_mtimes.get(p)] + if not changed_paths: + continue + + for path in changed_paths: + prev = last_mtimes.get(path, 0.0) + curr = current[path] + if curr == 0.0: + logger.info("provider_credentials.watcher file_removed path=%s", path) + elif prev == 0.0: + logger.info("provider_credentials.watcher file_appeared path=%s", path) + else: + logger.info("provider_credentials.watcher file_changed path=%s", path) + + last_mtimes = current + + try: + n = await seed(verbose=False) + if n: + logger.info("provider_credentials.watcher reseeded count=%d", n) + else: + logger.debug("provider_credentials.watcher reseed no_changes") + except Exception as exc: + logger.warning("provider_credentials.watcher reseed_failed error=%s", exc) + + except asyncio.CancelledError: + logger.info("provider_credentials.watcher stopped") + raise + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Seed or watch local AI provider credentials.") + parser.add_argument( + "--watch", + action="store_true", + help=f"Run continuously, re-seeding every {WATCH_INTERVAL_SECONDS}s when files change.", + ) + parser.add_argument( + "--interval", + type=int, + default=WATCH_INTERVAL_SECONDS, + metavar="SECONDS", + help=f"Poll interval in seconds (default: {WATCH_INTERVAL_SECONDS}).", + ) + args = parser.parse_args() + + if args.watch: + print(f"seed_provider_credentials: watching every {args.interval}s — Ctrl-C to stop") + asyncio.run(watch(interval=args.interval)) + else: + result = asyncio.run(seed(verbose=True)) + print(f"seed_provider_credentials: done — {result} row(s) affected") diff --git a/compose.yml b/compose.yml index 6448b09..f12f4e4 100644 --- a/compose.yml +++ b/compose.yml @@ -48,6 +48,11 @@ services: CLAUDE_CREDENTIALS_PATH: /run/secrets/claude_credentials # Codex CLI credentials — read-only mount for ChatGPT subscription usage auto-detection. CODEX_CREDENTIALS_PATH: /run/secrets/codex_credentials + # AI provider API keys — seeded into provider_credentials on boot (optional). + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} volumes: - ${CLAUDE_CREDENTIALS_FILE:-/home/kaspa/.claude/.credentials.json}:/run/secrets/claude_credentials:ro - ${CODEX_CREDENTIALS_FILE:-/home/kaspa/.codex/auth.json}:/run/secrets/codex_credentials:ro