diff --git a/CHANGELOG.md b/CHANGELOG.md index dc2009f..de4b781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to llm-relay are documented here. ## [Unreleased] ### Added +- **`llm-relay verify` command group** (`verify/`): four idempotent verification subcommands designed for agent-driven onboarding. `verify install` confirms the package itself (Python version, importability, entry points, optional extras, version consistency). `verify config` confirms local state (db dir, schema, writability, config file, knowledge dir, port availability, no deprecated env vars). `verify integration --cli {claude-code,openai-codex,gemini-cli,all}` confirms each CLI is wired through the relay (binary on PATH, settings file present, `ANTHROPIC_BASE_URL` routing to localhost, MCP server registered). `verify all` aggregates install + config + integration. Shared output schema (`schema_version: "1"`) emits per-check status (`pass`/`fail`/`warn`/`skipped`) with optional remediation strings; overall priority is `fail > warn > pass`. Skipped checks (e.g. CLI not installed) never escalate to fail. Optional `--live` probes `/_health` on the proxy. Common options: `--format {text,json}`, `--quiet`, `--no-remediation`. Exit code is 0 on pass/warn, 1 on fail. - **`llm-relay env-fingerprint` command** (`env_fingerprint.py`): single-shot, idempotent snapshot of the local LLM CLI environment for agent-driven onboarding. Emits a structured JSON (or YAML) document with sections for `llm_relay` package state, per-CLI install/auth/version, port availability, filesystem layout, redacted environment variables (API keys reported as `set`/`empty`/`null` only), and an optional `doctor` summary. Designed so an agent (Claude Code / Codex / Gemini) automating an llm-relay install can parse the environment instead of scraping `init` output. Schema versioned (`schema_version: "1"`); sub-probe failures surface as `_error` markers without crashing the snapshot. Options: `--format {json,yaml}`, `--no-doctor`, `--ports`. ### Performance diff --git a/src/llm_relay/detect/cli.py b/src/llm_relay/detect/cli.py index 0f9af1c..d8a8abf 100644 --- a/src/llm_relay/detect/cli.py +++ b/src/llm_relay/detect/cli.py @@ -278,6 +278,168 @@ def env_fingerprint(fmt: str, no_doctor: bool, ports_str: str) -> None: click.echo(json.dumps(snapshot, indent=2, ensure_ascii=False)) +# ── verify ────────────────────────────────────────────────────────────────── + + +_VERIFY_FORMAT_OPTION = click.option( + "--format", "fmt", + type=click.Choice(["text", "json"]), + default="text", + help="Output format (default: text, json for agent consumption).", +) +_VERIFY_QUIET_OPTION = click.option( + "--quiet", "-q", + is_flag=True, + help="Suppress checks that passed; show only warn/fail/skipped.", +) +_VERIFY_NO_REMEDIATION_OPTION = click.option( + "--no-remediation", + is_flag=True, + help="Omit remediation hints from output.", +) + + +def _render_verify_report(report, fmt: str, quiet: bool, no_remediation: bool) -> int: + """Render a VerifyReport. Returns the desired exit code.""" + import json as _json + + data = report.to_dict() + + if fmt == "json": + if no_remediation: + for c in data["checks"]: + c.pop("remediation", None) + if quiet: + data["checks"] = [c for c in data["checks"] if c["status"] != "pass"] + click.echo(_json.dumps(data, indent=2, ensure_ascii=False)) + else: + status_styles = {"pass": "green", "fail": "red", "warn": "yellow", "skipped": "blue"} + click.echo("target: {}".format(data["target"])) + click.echo("overall: {}".format( + click.style(data["overall"], fg=status_styles.get(data["overall"], "")) + )) + summary = data["summary"] + click.echo("summary: {}".format( + " ".join("{}={}".format(k, v) for k, v in summary.items()) + )) + click.echo("") + for c in data["checks"]: + if quiet and c["status"] == "pass": + continue + status_str = click.style( + "[{:7s}]".format(c["status"]), + fg=status_styles.get(c["status"], ""), + ) + click.echo("{} {}: {}".format(status_str, c["id"], c["detail"])) + if c.get("remediation") and not no_remediation: + click.echo(" → {}".format(c["remediation"])) + click.echo("") + + # Exit code: 0 on pass/warn, 1 on fail. + return 0 if data["overall"] != "fail" else 1 + + +@cli.group("verify") +def verify_group() -> None: + """Idempotent verification checks for install / config / integration. + + Designed to be consumed by an agent automating an llm-relay install: + `--format json` emits a stable schema (see src/llm_relay/verify/__init__.py). + + Exit code is 0 on pass/warn, 1 on fail. + """ + + +@verify_group.command("install") +@_VERIFY_FORMAT_OPTION +@_VERIFY_QUIET_OPTION +@_VERIFY_NO_REMEDIATION_OPTION +def verify_install_cmd(fmt: str, quiet: bool, no_remediation: bool) -> None: + """Verify the llm-relay package itself is correctly installed.""" + from llm_relay.verify.install import verify_install + + report = verify_install() + exit_code = _render_verify_report(report, fmt, quiet, no_remediation) + raise SystemExit(exit_code) + + +@verify_group.command("config") +@click.option("--port", default=8083, type=int, help="Proxy port to probe (default: 8083).") +@_VERIFY_FORMAT_OPTION +@_VERIFY_QUIET_OPTION +@_VERIFY_NO_REMEDIATION_OPTION +def verify_config_cmd(port: int, fmt: str, quiet: bool, no_remediation: bool) -> None: + """Verify the local llm-relay config (db, config file, port).""" + from llm_relay.verify.config import verify_config + + report = verify_config(port=port) + exit_code = _render_verify_report(report, fmt, quiet, no_remediation) + raise SystemExit(exit_code) + + +@verify_group.command("integration") +@click.option( + "--cli", "cli_id", + type=click.Choice(["claude-code", "openai-codex", "gemini-cli", "all"]), + default="all", + help="Which CLI integration to verify (default: all).", +) +@click.option( + "--live", + is_flag=True, + help="Also probe /_health on the proxy port (requires running server).", +) +@click.option("--port", default=8083, type=int, help="Proxy port for --live (default: 8083).") +@_VERIFY_FORMAT_OPTION +@_VERIFY_QUIET_OPTION +@_VERIFY_NO_REMEDIATION_OPTION +def verify_integration_cmd( + cli_id: str, + live: bool, + port: int, + fmt: str, + quiet: bool, + no_remediation: bool, +) -> None: + """Verify a CLI is wired through llm-relay (settings, proxy route, MCP).""" + from llm_relay.verify.integration import verify_integration + + report = verify_integration(cli_id, live=live, port=port) + exit_code = _render_verify_report(report, fmt, quiet, no_remediation) + raise SystemExit(exit_code) + + +@verify_group.command("all") +@click.option("--port", default=8083, type=int, help="Proxy port (default: 8083).") +@click.option("--live", is_flag=True, help="Include live proxy /_health probe.") +@_VERIFY_FORMAT_OPTION +@_VERIFY_QUIET_OPTION +@_VERIFY_NO_REMEDIATION_OPTION +def verify_all_cmd( + port: int, + live: bool, + fmt: str, + quiet: bool, + no_remediation: bool, +) -> None: + """Run install + config + integration (all CLIs) and aggregate.""" + from llm_relay.verify import aggregate + from llm_relay.verify.config import verify_config + from llm_relay.verify.install import verify_install + from llm_relay.verify.integration import verify_integration + + combined = aggregate( + "all", + [ + verify_install(), + verify_config(port=port), + verify_integration("all", live=live, port=port), + ], + ) + exit_code = _render_verify_report(combined, fmt, quiet, no_remediation) + raise SystemExit(exit_code) + + @cli.command() @click.option("--host", default="0.0.0.0", help="Bind address.") @click.option("--port", "-p", default=8083, type=int, help="Listen port.") diff --git a/src/llm_relay/verify/__init__.py b/src/llm_relay/verify/__init__.py new file mode 100644 index 0000000..dd25f85 --- /dev/null +++ b/src/llm_relay/verify/__init__.py @@ -0,0 +1,181 @@ +"""Verify primitives -- idempotent checks for install / config / integration. + +Each primitive returns a `VerifyReport`: a structured pass/fail/warn/skipped +record per check, designed to be consumed either by a human (`--format text`) +or by an agent automating an llm-relay install (`--format json`). + +This package complements `env_fingerprint`: env-fingerprint describes the +*state* of the user's environment, verify *asserts expectations* about that +state. An agent typically uses env-fingerprint to plan, then verify to +confirm each step it took. + +Shared output schema (schema_version "1"): + + { + "schema_version": "1", + "target": "install" | "config" | "integration", + "captured_at": "...", + "overall": "pass" | "fail" | "warn", + "summary": {"pass": N, "fail": N, "warn": N, "skipped": N}, + "checks": [ + { + "id": "...", + "label": "...", + "status": "pass" | "fail" | "warn" | "skipped", + "detail": "...", + "remediation": "..." | null, + "data": {...} | null + } + ] + } + +Status semantics: + pass -- expectation met + warn -- expectation met but with a caveat the operator/agent should know + fail -- expectation NOT met; remediation should fix it + skipped -- check was not applicable (e.g. CLI not installed for integration) + +Overall priority: fail > warn > (pass | skipped). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional + +SCHEMA_VERSION = "1" + +STATUS_PASS = "pass" +STATUS_FAIL = "fail" +STATUS_WARN = "warn" +STATUS_SKIPPED = "skipped" + +_ALL_STATUSES = (STATUS_PASS, STATUS_FAIL, STATUS_WARN, STATUS_SKIPPED) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +@dataclass +class VerifyCheck: + """A single verification result. + + `data` is optional structured detail that an agent may need (e.g. the + actual port number that was found in use). It's separate from `detail` + so machine consumers don't have to parse the human-friendly string. + """ + id: str + label: str + status: str + detail: str + remediation: Optional[str] = None + data: Optional[Dict[str, Any]] = None + + def __post_init__(self) -> None: + if self.status not in _ALL_STATUSES: + raise ValueError( + "VerifyCheck.status must be one of {} (got {!r})".format( + _ALL_STATUSES, self.status, + ) + ) + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "status": self.status, + "detail": self.detail, + "remediation": self.remediation, + "data": self.data, + } + + +@dataclass +class VerifyReport: + """Aggregate verification result for a single target.""" + target: str + captured_at: str = field(default_factory=_now_iso) + schema_version: str = SCHEMA_VERSION + checks: List[VerifyCheck] = field(default_factory=list) + + @property + def overall(self) -> str: + """fail > warn > pass. Skipped checks don't influence overall.""" + statuses = {c.status for c in self.checks} + if STATUS_FAIL in statuses: + return STATUS_FAIL + if STATUS_WARN in statuses: + return STATUS_WARN + return STATUS_PASS + + @property + def summary(self) -> Dict[str, int]: + counts = {status: 0 for status in _ALL_STATUSES} + for c in self.checks: + counts[c.status] += 1 + return counts + + def to_dict(self) -> Dict[str, Any]: + return { + "schema_version": self.schema_version, + "target": self.target, + "captured_at": self.captured_at, + "overall": self.overall, + "summary": self.summary, + "checks": [c.to_dict() for c in self.checks], + } + + +def run_check( + check_id: str, + label: str, + fn: Callable[[], VerifyCheck], + *, + fallback_remediation: Optional[str] = None, +) -> VerifyCheck: + """Run a check function, capturing exceptions as fail status. + + A check function returns a fully formed `VerifyCheck`. If the function + raises, we synthesize a fail result so a single broken probe never + crashes the whole report -- partial output beats no output. + """ + try: + result = fn() + except Exception as exc: # noqa: BLE001 -- intentional broad capture + return VerifyCheck( + id=check_id, + label=label, + status=STATUS_FAIL, + detail="{}: {}".format(type(exc).__name__, exc), + remediation=fallback_remediation, + data={"_error": True}, + ) + # Sanity: enforce id/label consistency in case the check function returns + # an unrelated VerifyCheck by mistake. The provided id/label always win. + result.id = check_id + result.label = label + return result + + +def aggregate(target: str, reports: List[VerifyReport]) -> VerifyReport: + """Combine multiple sub-reports into one (e.g. for `verify all`). + + The target string identifies the combined report (e.g. "all" or + "integration" when collapsing per-CLI sub-reports). Check IDs from + sub-reports are namespaced as `{sub_target}.{original_id}` to avoid + collisions across sub-reports. + """ + combined = VerifyReport(target=target) + for sub in reports: + for check in sub.checks: + combined.checks.append(VerifyCheck( + id="{}.{}".format(sub.target, check.id), + label=check.label, + status=check.status, + detail=check.detail, + remediation=check.remediation, + data=check.data, + )) + return combined diff --git a/src/llm_relay/verify/config.py b/src/llm_relay/verify/config.py new file mode 100644 index 0000000..c4891de --- /dev/null +++ b/src/llm_relay/verify/config.py @@ -0,0 +1,263 @@ +"""verify config -- confirm local llm-relay state (DB, config files, ports). + +Checks examine `~/.llm-relay/` (or whatever `db_dir_for_env()` returns) and +neighbouring filesystem state. They are read-only with one exception: +`db_writable` opens a transaction, inserts a marker row, then rolls back -- +this is the only reliable way to detect a read-only mount or quota issue. +""" + +from __future__ import annotations + +import os +import sqlite3 +from pathlib import Path + +from llm_relay.setup_init import _is_port_in_use, _read_json, db_dir_for_env +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_WARN, + VerifyCheck, + VerifyReport, + run_check, +) + +# Tables we expect after a successful `llm-relay init`. +_REQUIRED_TABLES = {"requests"} + +# Env vars that were renamed during the cc-relay → llm-relay merge. +# Their presence is not destructive, but they're ignored by current code +# and silently shadowed by their LLM_RELAY_* equivalents. +_DEPRECATED_ENV_PREFIXES = ("CCPULSE_", "CC_RELAY_") + +_DEFAULT_PROXY_PORT = 8083 + + +def _check_db_dir_exists() -> VerifyCheck: + db_dir = db_dir_for_env() + if db_dir.is_dir(): + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="db dir exists at {}".format(db_dir), + data={"path": str(db_dir)}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="db dir not found at {}".format(db_dir), + remediation="Run `llm-relay init` to create the directory.", + data={"path": str(db_dir)}, + ) + + +def _check_db_initialized() -> VerifyCheck: + """usage.db exists AND has the expected table schema.""" + db_path = db_dir_for_env() / "usage.db" + if not db_path.is_file(): + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="usage.db not found at {}".format(db_path), + remediation="Run `llm-relay init` to initialize the database.", + data={"path": str(db_path)}, + ) + try: + conn = sqlite3.connect(str(db_path)) + try: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + present = {row[0] for row in rows} + finally: + conn.close() + except sqlite3.Error as exc: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="failed to open usage.db: {}".format(exc), + remediation="Run `llm-relay init` (the file may be corrupt).", + data={"path": str(db_path)}, + ) + missing = _REQUIRED_TABLES - present + if missing: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="usage.db is missing tables: {}".format(", ".join(sorted(missing))), + remediation="Run `llm-relay init` to recreate the schema.", + data={"missing_tables": sorted(missing), "present_tables": sorted(present)}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="usage.db schema OK ({} table(s) present)".format(len(present)), + data={"present_tables": sorted(present)}, + ) + + +def _check_db_writable() -> VerifyCheck: + """Round-trip a transaction to detect read-only mounts / quota issues.""" + db_path = db_dir_for_env() / "usage.db" + if not db_path.is_file(): + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="usage.db not found (cannot test write)", + remediation="Run `llm-relay init` first.", + ) + try: + conn = sqlite3.connect(str(db_path)) + try: + conn.execute("BEGIN") + conn.execute( + "CREATE TABLE IF NOT EXISTS _verify_probe (ts INTEGER PRIMARY KEY)" + ) + conn.execute("INSERT INTO _verify_probe (ts) VALUES (?)", (1,)) + conn.execute("ROLLBACK") + finally: + conn.close() + except sqlite3.Error as exc: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="db write probe failed: {}".format(exc), + remediation="Check filesystem permissions and free space on the db dir.", + ) + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="db accepts writes (probe transaction rolled back)", + ) + + +def _check_config_file() -> VerifyCheck: + """config.json is optional but expected after init.""" + config_path = db_dir_for_env() / "config.json" + if not config_path.is_file(): + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="config.json not found at {}".format(config_path), + remediation="Run `llm-relay init` to generate the config file.", + data={"path": str(config_path)}, + ) + try: + data = _read_json(config_path) + except Exception as exc: # noqa: BLE001 + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="config.json present but unparseable: {}".format(exc), + remediation="Re-run `llm-relay init` or delete the file and let init recreate it.", + ) + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="config.json parseable ({} top-level keys)".format(len(data)), + data={"path": str(config_path), "keys": sorted(data.keys())}, + ) + + +def _check_knowledge_dir() -> VerifyCheck: + """Knowledge directory is optional (only used if the knowledge module + is enabled). Missing → warn, not fail.""" + knowledge_dir = db_dir_for_env() / "knowledge" + if knowledge_dir.is_dir(): + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="knowledge dir exists at {}".format(knowledge_dir), + data={"path": str(knowledge_dir)}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="knowledge dir not found at {}".format(knowledge_dir), + remediation="Run `llm-relay init` (knowledge dir is auto-created).", + data={"path": str(knowledge_dir)}, + ) + + +def _check_port_available(port: int = _DEFAULT_PROXY_PORT) -> VerifyCheck: + """Default proxy port should be either free, or already bound by our own + server. We can't distinguish those without a deeper probe, so we report + `warn` when the port is busy and let the operator/agent decide. + """ + if not _is_port_in_use(port): + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="proxy port {} is free".format(port), + data={"port": port, "in_use": False}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="proxy port {} is already bound (could be our own server)".format(port), + remediation="Pass `--port ` to `llm-relay serve` or stop the conflicting process.", + data={"port": port, "in_use": True}, + ) + + +def _check_no_deprecated_env() -> VerifyCheck: + """CCPULSE_* / CC_RELAY_* env vars are silently ignored by current code + and indicate a stale shell profile -- worth flagging so the operator + cleans them up. + """ + found = [ + name for name in os.environ + if any(name.startswith(prefix) for prefix in _DEPRECATED_ENV_PREFIXES) + ] + if not found: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="no deprecated env vars set", + ) + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="deprecated env vars set: {}".format(", ".join(sorted(found))), + remediation=( + "Rename `CCPULSE_*` / `CC_RELAY_*` to `LLM_RELAY_*` in your shell rc " + "or remove them. Current code ignores the legacy names." + ), + data={"found": sorted(found)}, + ) + + +def _check(check_id, label, fn): + """Local convenience to keep the registration table compact.""" + return (check_id, label, fn) + + +_CHECKS = [ + _check("db_dir_exists", "llm-relay db directory exists", _check_db_dir_exists), + _check("db_initialized", "usage.db schema is initialized", _check_db_initialized), + _check("db_writable", "usage.db accepts writes", _check_db_writable), + _check("config_file", "config.json is present and parseable", _check_config_file), + _check("knowledge_dir", "knowledge directory exists (optional)", _check_knowledge_dir), + _check("port_available", "default proxy port is usable", _check_port_available), + _check("no_deprecated_env", "no CCPULSE_*/CC_RELAY_* env vars set", _check_no_deprecated_env), +] + + +def verify_config(*, port: int = _DEFAULT_PROXY_PORT) -> VerifyReport: + """Run all config-time checks and return a structured report. + + `port` is the proxy port to probe for availability (default 8083). + """ + report = VerifyReport(target="config") + for check_id, label, fn in _CHECKS: + if check_id == "port_available": + report.checks.append( + run_check(check_id, label, lambda p=port: _check_port_available(p)) + ) + else: + report.checks.append(run_check(check_id, label, fn)) + return report + + +# Re-export for tests that want to call the underlying helpers directly. +__all__ = ["verify_config", "Path"] diff --git a/src/llm_relay/verify/install.py b/src/llm_relay/verify/install.py new file mode 100644 index 0000000..dc55f1f --- /dev/null +++ b/src/llm_relay/verify/install.py @@ -0,0 +1,191 @@ +"""verify install -- confirm the llm-relay package itself is usable. + +Checks are ordered from most-fundamental (Python version) to most-optional +(MCP entry point). An agent reading the report should be able to act on the +remediation strings directly. +""" + +from __future__ import annotations + +import importlib +import shutil +import sys +from importlib.metadata import PackageNotFoundError, version + +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_SKIPPED, + STATUS_WARN, + VerifyCheck, + VerifyReport, + run_check, +) + +_MIN_PYTHON = (3, 9) + + +def _check_python_version() -> VerifyCheck: + cur = sys.version_info[:3] + if cur[:2] >= _MIN_PYTHON: + return VerifyCheck( + id="", label="", # filled by run_check + status=STATUS_PASS, + detail="Python {}.{}.{}".format(*cur), + data={"required": "{}.{}+".format(*_MIN_PYTHON), "current": ".".join(map(str, cur))}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="Python {}.{}.{} is below the required {}.{}+".format(*cur, *_MIN_PYTHON), + remediation="Install Python {}.{} or newer.".format(*_MIN_PYTHON), + data={"required": "{}.{}+".format(*_MIN_PYTHON), "current": ".".join(map(str, cur))}, + ) + + +def _check_package_importable() -> VerifyCheck: + try: + mod = importlib.import_module("llm_relay") + except ImportError as exc: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="import llm_relay failed: {}".format(exc), + remediation="pip install llm-relay", + ) + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="llm_relay imported from {}".format(getattr(mod, "__file__", "(builtin)")), + data={"module_file": getattr(mod, "__file__", None)}, + ) + + +def _check_entry_point_relay() -> VerifyCheck: + path = shutil.which("llm-relay") + if path: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="llm-relay entry point at {}".format(path), + data={"path": path}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="llm-relay binary not found on PATH", + remediation=( + "Reinstall the package (`pip install --force-reinstall llm-relay`) " + "or ensure your Python scripts directory is on PATH." + ), + ) + + +def _check_entry_point_mcp() -> VerifyCheck: + """MCP entry point is an optional extra. Missing == warn, not fail.""" + path = shutil.which("llm-relay-mcp") + if path: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="llm-relay-mcp entry point at {}".format(path), + data={"path": path}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="llm-relay-mcp binary not found (optional MCP extra)", + remediation="pip install llm-relay[mcp] (only needed if exposing the MCP server)", + ) + + +def _check_proxy_extras() -> VerifyCheck: + """Proxy extras are optional; missing means the [proxy] feature won't work.""" + missing = [] + for module_name in ("httpx", "uvicorn", "starlette"): + try: + importlib.import_module(module_name) + except ImportError: + missing.append(module_name) + if not missing: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="proxy extras importable (httpx, uvicorn, starlette)", + ) + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="proxy extras missing: {}".format(", ".join(missing)), + remediation="pip install llm-relay[proxy] (needed for `llm-relay serve`)", + data={"missing": missing}, + ) + + +def _check_version_consistency() -> VerifyCheck: + """The installed wheel's metadata version should match what the running + process is using. A mismatch usually means a stale editable install. + """ + try: + meta_ver = version("llm-relay") + except PackageNotFoundError: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="package metadata not found (importlib.metadata.PackageNotFoundError)", + remediation="pip install llm-relay (the package is imported but not installed?)", + ) + try: + mod = importlib.import_module("llm_relay") + except ImportError: + # Package metadata exists but import failed -- the importable check + # above already covered that; skip here. + return VerifyCheck( + id="", label="", + status=STATUS_SKIPPED, + detail="cannot compare versions: import failed (see package_importable)", + ) + runtime_ver = getattr(mod, "__version__", None) + if runtime_ver is None: + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="package metadata version is {} but llm_relay.__version__ is unset".format(meta_ver), + data={"metadata_version": meta_ver, "runtime_version": None}, + ) + if runtime_ver != meta_ver: + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="metadata version {!r} != runtime __version__ {!r} (stale install?)".format( + meta_ver, runtime_ver, + ), + remediation="pip install --force-reinstall llm-relay", + data={"metadata_version": meta_ver, "runtime_version": runtime_ver}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="version {} (consistent across metadata and runtime)".format(meta_ver), + data={"metadata_version": meta_ver, "runtime_version": runtime_ver}, + ) + + +_CHECKS = [ + ("python_version", "Python interpreter version", _check_python_version, None), + ("package_importable", "llm-relay package can be imported", _check_package_importable, None), + ("entry_point_relay", "llm-relay CLI entry point exists", _check_entry_point_relay, None), + ("entry_point_mcp", "llm-relay-mcp entry point exists (optional)", _check_entry_point_mcp, None), + ("proxy_extras", "proxy extras (httpx/uvicorn/starlette) importable", _check_proxy_extras, None), + ("version_consistency", "package metadata and runtime version agree", _check_version_consistency, None), +] + + +def verify_install() -> VerifyReport: + """Run all install-time checks and return a structured report.""" + report = VerifyReport(target="install") + for check_id, label, fn, fallback_rem in _CHECKS: + report.checks.append( + run_check(check_id, label, fn, fallback_remediation=fallback_rem) + ) + return report diff --git a/src/llm_relay/verify/integration.py b/src/llm_relay/verify/integration.py new file mode 100644 index 0000000..da178f2 --- /dev/null +++ b/src/llm_relay/verify/integration.py @@ -0,0 +1,372 @@ +"""verify integration -- confirm a target LLM CLI is wired to the relay. + +For each CLI we check (a) the binary is on PATH, (b) its config file is +present and parseable, and (c) the relay-specific settings are in place +(ANTHROPIC_BASE_URL, MCP server registration, etc.). + +CLIs that aren't installed produce `skipped` checks rather than `fail`, +since the user may legitimately use only one CLI. + +Known limitations preserved as `warn`s: + - Gemini CLI oauth-personal hits an upstream 403 (#25425); we surface + this as a known-issue warning rather than a failure. + - Codex CLI does not have a stable proxy-routing knob yet; the + proxy_route check is `skipped` for it. + +Each CLI runs in its own sub-report; `verify_integration("all")` aggregates +them via `verify.aggregate()`, so check IDs collide-free. +""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path +from typing import Callable, Dict, List, Optional, Tuple + +from llm_relay.setup_init import _read_json +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_SKIPPED, + STATUS_WARN, + VerifyCheck, + VerifyReport, + aggregate, + run_check, +) + +# (cli_id, binary_name, config_path_relative_to_home) +_CLI_REGISTRY: List[Tuple[str, str, str]] = [ + ("claude-code", "claude", ".claude/settings.json"), + ("openai-codex", "codex", ".codex/config.toml"), + ("gemini-cli", "gemini", ".gemini"), +] + +_REGISTRY_BY_ID = {row[0]: row for row in _CLI_REGISTRY} + +ALL_CLIS = "all" + + +def _check_binary(binary_name: str) -> VerifyCheck: + path = shutil.which(binary_name) + if path: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="{} on PATH at {}".format(binary_name, path), + data={"path": path}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="{} binary not found on PATH".format(binary_name), + remediation="Install {} (see vendor docs).".format(binary_name), + ) + + +def _check_config_present(label_path: str, abs_path: Path) -> VerifyCheck: + if abs_path.exists(): + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="{} present at {}".format(label_path, abs_path), + data={"path": str(abs_path)}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="{} not found at {}".format(label_path, abs_path), + remediation="Launch the CLI once to let it create its config, then re-run.", + data={"path": str(abs_path)}, + ) + + +# ── claude-code ───────────────────────────────────────────────────────────── + + +def _claude_settings_path() -> Path: + return Path.home() / ".claude" / "settings.json" + + +def _claude_check_settings_present() -> VerifyCheck: + settings_path = _claude_settings_path() + check = _check_config_present("Claude settings.json", settings_path) + if check.status != STATUS_PASS: + return check + # Additionally confirm parseability + try: + _read_json(settings_path) + except Exception as exc: # noqa: BLE001 + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="settings.json present but unparseable: {}".format(exc), + remediation=( + "Repair the JSON manually, or delete settings.json and let " + "Claude Code regenerate it (your hooks/permissions will reset)." + ), + data={"path": str(settings_path)}, + ) + return check + + +def _claude_check_proxy_route() -> VerifyCheck: + settings_path = _claude_settings_path() + if not settings_path.is_file(): + return VerifyCheck( + id="", label="", + status=STATUS_SKIPPED, + detail="settings.json missing (see claude_settings_present)", + ) + settings = _read_json(settings_path) + env = settings.get("env", {}) if isinstance(settings, dict) else {} + base_url = env.get("ANTHROPIC_BASE_URL") if isinstance(env, dict) else None + env_base_url = os.environ.get("ANTHROPIC_BASE_URL") + effective = env_base_url or base_url + if effective and ("localhost" in effective or "127.0.0.1" in effective): + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="ANTHROPIC_BASE_URL routes to local relay: {}".format(effective), + data={"source": "env" if env_base_url else "settings.json", "value": effective}, + ) + if effective: + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail="ANTHROPIC_BASE_URL set but not local: {}".format(effective), + remediation=( + "Run `llm-relay init` or set ANTHROPIC_BASE_URL=http://localhost:8083 " + "if you want Claude Code to route through llm-relay." + ), + data={"source": "env" if env_base_url else "settings.json", "value": effective}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="ANTHROPIC_BASE_URL not configured", + remediation="Run `llm-relay init` to wire Claude Code through the relay.", + ) + + +def _claude_check_mcp_registered() -> VerifyCheck: + settings_path = _claude_settings_path() + if not settings_path.is_file(): + return VerifyCheck( + id="", label="", + status=STATUS_SKIPPED, + detail="settings.json missing (see claude_settings_present)", + ) + settings = _read_json(settings_path) + mcp = settings.get("mcpServers", {}) if isinstance(settings, dict) else {} + if isinstance(mcp, dict) and "llm-relay" in mcp: + entry = mcp["llm-relay"] + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="llm-relay MCP server registered", + data={"entry": entry}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="llm-relay MCP server not registered in settings.json", + remediation="Run `llm-relay init` (registers llm-relay-mcp as a stdio server).", + ) + + +def _claude_checks() -> List[Tuple[str, str, Callable[[], VerifyCheck]]]: + return [ + ("binary", "claude binary on PATH", lambda: _check_binary("claude")), + ("settings_present", "~/.claude/settings.json present and parseable", + _claude_check_settings_present), + ("proxy_route", "ANTHROPIC_BASE_URL routes to local relay", + _claude_check_proxy_route), + ("mcp_server", "llm-relay MCP server registered in settings.json", + _claude_check_mcp_registered), + ] + + +# ── openai-codex ──────────────────────────────────────────────────────────── + + +def _codex_config_path() -> Path: + return Path.home() / ".codex" / "config.toml" + + +def _codex_check_config_present() -> VerifyCheck: + return _check_config_present("Codex config.toml", _codex_config_path()) + + +def _codex_check_proxy_route() -> VerifyCheck: + """Codex doesn't yet have a stable proxy-routing knob exposed via config, + so we surface this as `skipped` with a note so agents don't try to wire + Codex into the relay until upstream support lands. + """ + return VerifyCheck( + id="", label="", + status=STATUS_SKIPPED, + detail=( + "Codex CLI does not currently expose a stable proxy-routing setting; " + "relay-routed Codex calls are out of scope for this check." + ), + data={"reason": "upstream"}, + ) + + +def _codex_checks() -> List[Tuple[str, str, Callable[[], VerifyCheck]]]: + return [ + ("binary", "codex binary on PATH", lambda: _check_binary("codex")), + ("config_present", "~/.codex/config.toml present", _codex_check_config_present), + ("proxy_route", "Codex proxy routing (upstream limitation)", _codex_check_proxy_route), + ] + + +# ── gemini-cli ────────────────────────────────────────────────────────────── + + +def _gemini_dir() -> Path: + return Path.home() / ".gemini" + + +def _gemini_check_dir_present() -> VerifyCheck: + return _check_config_present("~/.gemini directory", _gemini_dir()) + + +def _gemini_check_oauth_known_issue() -> VerifyCheck: + """We always surface the known oauth-personal 403 bug as a `warn` so the + operator/agent knows to fall back to GEMINI_API_KEY if they hit it. + """ + return VerifyCheck( + id="", label="", + status=STATUS_WARN, + detail=( + "Gemini CLI oauth-personal has a known 403 server-side bug " + "(google-gemini/gemini-cli#25425). Use GEMINI_API_KEY to bypass." + ), + remediation="export GEMINI_API_KEY= # or use a service-account flow", + data={"upstream_issue": "google-gemini/gemini-cli#25425"}, + ) + + +def _gemini_checks() -> List[Tuple[str, str, Callable[[], VerifyCheck]]]: + return [ + ("binary", "gemini binary on PATH", lambda: _check_binary("gemini")), + ("config_dir_present", "~/.gemini directory present", _gemini_check_dir_present), + ("oauth_known_issue", "oauth-personal upstream 403 known-issue note", + _gemini_check_oauth_known_issue), + ] + + +# ── proxy_reachable (optional, --live) ────────────────────────────────────── + + +def _live_check_proxy_reachable(port: int) -> VerifyCheck: + """Hit the relay's /_health endpoint to confirm the proxy is actually + answering on `port`. Optional because it requires httpx and a running + server. + """ + try: + import httpx # type: ignore[import-not-found] + except ImportError: + return VerifyCheck( + id="", label="", + status=STATUS_SKIPPED, + detail="httpx not installed; pass [proxy] extra to enable --live check", + ) + url = "http://127.0.0.1:{}/_health".format(port) + try: + resp = httpx.get(url, timeout=2.0) + except httpx.HTTPError as exc: + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="proxy /_health unreachable at {}: {}".format(url, exc), + remediation="Start the relay (`llm-relay serve --port {}`).".format(port), + data={"url": url}, + ) + if resp.status_code == 200: + return VerifyCheck( + id="", label="", + status=STATUS_PASS, + detail="proxy /_health responded 200 at {}".format(url), + data={"url": url, "status_code": 200}, + ) + return VerifyCheck( + id="", label="", + status=STATUS_FAIL, + detail="proxy /_health returned status {} at {}".format(resp.status_code, url), + remediation="Inspect server logs (`journalctl --user -u llm-relay-api`).", + data={"url": url, "status_code": resp.status_code}, + ) + + +# ── Dispatch ──────────────────────────────────────────────────────────────── + + +_CHECKS_BY_CLI: Dict[str, Callable[[], List[Tuple[str, str, Callable[[], VerifyCheck]]]]] = { + "claude-code": _claude_checks, + "openai-codex": _codex_checks, + "gemini-cli": _gemini_checks, +} + + +def _verify_one_cli(cli_id: str, *, live: bool, port: int) -> VerifyReport: + if cli_id not in _CHECKS_BY_CLI: + raise ValueError("Unknown cli_id {!r}; expected one of {}".format( + cli_id, sorted(list(_CHECKS_BY_CLI.keys()) + [ALL_CLIS]), + )) + binary_name = _REGISTRY_BY_ID[cli_id][1] + binary_present = shutil.which(binary_name) is not None + report = VerifyReport(target=cli_id) + + if not binary_present: + # When the binary isn't installed, every check for this CLI is + # `skipped` -- the user may not use this CLI at all. + for check_id, label, _ in _CHECKS_BY_CLI[cli_id](): + report.checks.append(VerifyCheck( + id=check_id, + label=label, + status=STATUS_SKIPPED, + detail="{} not installed".format(binary_name), + )) + return report + + for check_id, label, fn in _CHECKS_BY_CLI[cli_id](): + report.checks.append(run_check(check_id, label, fn)) + + if live: + report.checks.append( + run_check( + "proxy_reachable_live", + "proxy /_health responds on configured port", + lambda: _live_check_proxy_reachable(port), + ) + ) + return report + + +def verify_integration( + cli_id: Optional[str] = ALL_CLIS, + *, + live: bool = False, + port: int = 8083, +) -> VerifyReport: + """Verify a single CLI or all of them. + + `cli_id="all"` aggregates per-CLI sub-reports into one combined report; + each check ID is namespaced as `{cli_id}.{original_id}`. + """ + if cli_id is None: + cli_id = ALL_CLIS + if cli_id == ALL_CLIS: + sub_reports = [ + _verify_one_cli(c, live=live, port=port) + for c in _CHECKS_BY_CLI + ] + return aggregate("integration", sub_reports) + return _verify_one_cli(cli_id, live=live, port=port) + + +__all__ = ["verify_integration", "ALL_CLIS"] diff --git a/tests/test_verify/__init__.py b/tests/test_verify/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_verify/test_common.py b/tests/test_verify/test_common.py new file mode 100644 index 0000000..56fae2e --- /dev/null +++ b/tests/test_verify/test_common.py @@ -0,0 +1,130 @@ +"""Tests for the shared verify dataclasses and helpers.""" + +from __future__ import annotations + +import pytest + +from llm_relay.verify import ( + SCHEMA_VERSION, + STATUS_FAIL, + STATUS_PASS, + STATUS_SKIPPED, + STATUS_WARN, + VerifyCheck, + VerifyReport, + aggregate, + run_check, +) + + +class TestVerifyCheck: + def test_rejects_invalid_status(self): + with pytest.raises(ValueError): + VerifyCheck(id="x", label="x", status="bogus", detail="...") + + def test_to_dict_round_trip(self): + c = VerifyCheck( + id="x", label="X check", status=STATUS_PASS, detail="ok", + remediation="do X", data={"k": 1}, + ) + d = c.to_dict() + assert d == { + "id": "x", "label": "X check", "status": "pass", + "detail": "ok", "remediation": "do X", "data": {"k": 1}, + } + + +class TestVerifyReport: + def test_empty_report_is_pass(self): + r = VerifyReport(target="install") + assert r.overall == STATUS_PASS + assert r.summary == {"pass": 0, "fail": 0, "warn": 0, "skipped": 0} + + def test_overall_priority_fail_over_warn(self): + r = VerifyReport(target="install") + r.checks.append(VerifyCheck(id="a", label="a", status=STATUS_WARN, detail="")) + r.checks.append(VerifyCheck(id="b", label="b", status=STATUS_FAIL, detail="")) + r.checks.append(VerifyCheck(id="c", label="c", status=STATUS_PASS, detail="")) + assert r.overall == STATUS_FAIL + + def test_overall_warn_when_no_fail(self): + r = VerifyReport(target="install") + r.checks.append(VerifyCheck(id="a", label="a", status=STATUS_WARN, detail="")) + r.checks.append(VerifyCheck(id="b", label="b", status=STATUS_PASS, detail="")) + r.checks.append(VerifyCheck(id="c", label="c", status=STATUS_SKIPPED, detail="")) + assert r.overall == STATUS_WARN + + def test_skipped_does_not_affect_overall(self): + r = VerifyReport(target="install") + r.checks.append(VerifyCheck(id="a", label="a", status=STATUS_SKIPPED, detail="")) + r.checks.append(VerifyCheck(id="b", label="b", status=STATUS_PASS, detail="")) + assert r.overall == STATUS_PASS + + def test_summary_counts_by_status(self): + r = VerifyReport(target="install") + for status in (STATUS_PASS, STATUS_PASS, STATUS_WARN, STATUS_FAIL, STATUS_SKIPPED): + r.checks.append(VerifyCheck(id="x", label="x", status=status, detail="")) + assert r.summary == {"pass": 2, "fail": 1, "warn": 1, "skipped": 1} + + def test_to_dict_includes_schema_version(self): + r = VerifyReport(target="install") + d = r.to_dict() + assert d["schema_version"] == SCHEMA_VERSION + assert d["target"] == "install" + assert d["overall"] == "pass" + assert "captured_at" in d + assert "summary" in d + assert d["checks"] == [] + + def test_captured_at_is_iso_with_tz(self): + from datetime import datetime + r = VerifyReport(target="install") + parsed = datetime.fromisoformat(r.captured_at) + assert parsed.tzinfo is not None + + +class TestRunCheck: + def test_captures_exception_as_fail(self): + def boom(): + raise RuntimeError("kaboom") + + c = run_check("x", "X", boom, fallback_remediation="retry") + assert c.status == STATUS_FAIL + assert c.id == "x" + assert c.label == "X" + assert "RuntimeError" in c.detail + assert "kaboom" in c.detail + assert c.remediation == "retry" + assert c.data == {"_error": True} + + def test_passes_through_normal_result(self): + def ok(): + return VerifyCheck( + id="", label="", status=STATUS_PASS, detail="all good", + ) + + c = run_check("real_id", "Real label", ok) + assert c.status == STATUS_PASS + assert c.id == "real_id" # id/label always wins + assert c.label == "Real label" + assert c.detail == "all good" + + +class TestAggregate: + def test_namespaces_check_ids(self): + sub_a = VerifyReport(target="install") + sub_a.checks.append(VerifyCheck(id="x", label="X", status=STATUS_PASS, detail="")) + sub_b = VerifyReport(target="config") + sub_b.checks.append(VerifyCheck(id="x", label="X", status=STATUS_FAIL, detail="")) + + combined = aggregate("all", [sub_a, sub_b]) + ids = {c.id for c in combined.checks} + # Same original id "x" but namespaced by sub-target -- no collision + assert ids == {"install.x", "config.x"} + assert combined.overall == STATUS_FAIL + + def test_target_label_preserved(self): + sub = VerifyReport(target="install") + sub.checks.append(VerifyCheck(id="x", label="X", status=STATUS_PASS, detail="")) + combined = aggregate("custom-label", [sub]) + assert combined.target == "custom-label" diff --git a/tests/test_verify/test_config.py b/tests/test_verify/test_config.py new file mode 100644 index 0000000..fa59478 --- /dev/null +++ b/tests/test_verify/test_config.py @@ -0,0 +1,146 @@ +"""Tests for verify config checks.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_WARN, +) +from llm_relay.verify.config import verify_config + + +def _make_initialized_db_dir(root: Path) -> Path: + """Create a fake db dir with usage.db containing the requests table.""" + db_dir = root / ".llm-relay" + db_dir.mkdir(parents=True) + db_path = db_dir / "usage.db" + conn = sqlite3.connect(str(db_path)) + try: + conn.execute( + "CREATE TABLE requests (id INTEGER PRIMARY KEY, ts REAL, session_id TEXT)" + ) + conn.commit() + finally: + conn.close() + return db_dir + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + """Redirect db_dir_for_env to a temp dir for isolation.""" + fake = tmp_path / "home" + fake.mkdir() + monkeypatch.setattr( + "llm_relay.verify.config.db_dir_for_env", + lambda: fake / ".llm-relay", + ) + return fake + + +class TestVerifyConfig: + def test_returns_report_with_config_target(self, fake_home): + report = verify_config() + assert report.target == "config" + + def test_runs_all_seven_checks(self, fake_home): + report = verify_config() + check_ids = {c.id for c in report.checks} + assert check_ids == { + "db_dir_exists", + "db_initialized", + "db_writable", + "config_file", + "knowledge_dir", + "port_available", + "no_deprecated_env", + } + + def test_db_dir_missing_fails(self, fake_home): + report = verify_config() + check = next(c for c in report.checks if c.id == "db_dir_exists") + assert check.status == STATUS_FAIL + assert "Run `llm-relay init`" in check.remediation + + def test_db_dir_exists_when_initialized(self, fake_home): + _make_initialized_db_dir(fake_home) + report = verify_config() + check = next(c for c in report.checks if c.id == "db_dir_exists") + assert check.status == STATUS_PASS + + def test_db_initialized_pass(self, fake_home): + _make_initialized_db_dir(fake_home) + report = verify_config() + check = next(c for c in report.checks if c.id == "db_initialized") + assert check.status == STATUS_PASS + assert "requests" in check.data["present_tables"] + + def test_db_initialized_fail_without_requests_table(self, fake_home): + db_dir = fake_home / ".llm-relay" + db_dir.mkdir(parents=True) + # Create an empty db without the requests table + conn = sqlite3.connect(str(db_dir / "usage.db")) + conn.close() + report = verify_config() + check = next(c for c in report.checks if c.id == "db_initialized") + assert check.status == STATUS_FAIL + assert "requests" in check.data["missing_tables"] + + def test_db_writable_pass(self, fake_home): + _make_initialized_db_dir(fake_home) + report = verify_config() + check = next(c for c in report.checks if c.id == "db_writable") + assert check.status == STATUS_PASS + + def test_config_file_warn_when_missing(self, fake_home): + _make_initialized_db_dir(fake_home) + report = verify_config() + check = next(c for c in report.checks if c.id == "config_file") + assert check.status == STATUS_WARN + + def test_config_file_pass_when_present(self, fake_home): + db_dir = _make_initialized_db_dir(fake_home) + (db_dir / "config.json").write_text(json.dumps({"port": 8083})) + report = verify_config() + check = next(c for c in report.checks if c.id == "config_file") + assert check.status == STATUS_PASS + assert "port" in check.data["keys"] + + def test_knowledge_dir_warn_when_missing(self, fake_home): + _make_initialized_db_dir(fake_home) + report = verify_config() + check = next(c for c in report.checks if c.id == "knowledge_dir") + assert check.status == STATUS_WARN + + def test_knowledge_dir_pass_when_present(self, fake_home): + db_dir = _make_initialized_db_dir(fake_home) + (db_dir / "knowledge").mkdir() + report = verify_config() + check = next(c for c in report.checks if c.id == "knowledge_dir") + assert check.status == STATUS_PASS + + def test_no_deprecated_env_pass_when_clean(self, fake_home, monkeypatch): + for name in list(__import__("os").environ): + if name.startswith(("CCPULSE_", "CC_RELAY_")): + monkeypatch.delenv(name, raising=False) + report = verify_config() + check = next(c for c in report.checks if c.id == "no_deprecated_env") + assert check.status == STATUS_PASS + + def test_no_deprecated_env_warn_when_legacy_set(self, fake_home, monkeypatch): + monkeypatch.setenv("CCPULSE_DEBUG", "1") + report = verify_config() + check = next(c for c in report.checks if c.id == "no_deprecated_env") + assert check.status == STATUS_WARN + assert "CCPULSE_DEBUG" in check.data["found"] + + def test_port_available_check_data_has_port(self, fake_home): + report = verify_config(port=59999) # extremely unlikely to be bound + check = next(c for c in report.checks if c.id == "port_available") + assert check.data["port"] == 59999 diff --git a/tests/test_verify/test_install.py b/tests/test_verify/test_install.py new file mode 100644 index 0000000..47b7304 --- /dev/null +++ b/tests/test_verify/test_install.py @@ -0,0 +1,92 @@ +"""Tests for verify install checks.""" + +from __future__ import annotations + +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_WARN, +) +from llm_relay.verify.install import verify_install + + +class TestVerifyInstall: + def test_returns_report_with_install_target(self): + report = verify_install() + assert report.target == "install" + + def test_runs_all_six_checks(self): + report = verify_install() + check_ids = {c.id for c in report.checks} + assert check_ids == { + "python_version", + "package_importable", + "entry_point_relay", + "entry_point_mcp", + "proxy_extras", + "version_consistency", + } + + def test_python_version_passes_on_supported_runtime(self): + # We run our tests on Python >= 3.9; this check should always pass. + report = verify_install() + py = next(c for c in report.checks if c.id == "python_version") + assert py.status == STATUS_PASS + assert "current" in py.data + assert "required" in py.data + + def test_package_importable_passes_for_self(self): + report = verify_install() + check = next(c for c in report.checks if c.id == "package_importable") + assert check.status == STATUS_PASS + assert check.data["module_file"] is not None + + def test_entry_point_mcp_warn_when_missing(self, monkeypatch): + # shutil.which is the only thing that determines this + def fake_which(name): + if name == "llm-relay-mcp": + return None + return "/fake/{}".format(name) + + monkeypatch.setattr("llm_relay.verify.install.shutil.which", fake_which) + report = verify_install() + check = next(c for c in report.checks if c.id == "entry_point_mcp") + assert check.status == STATUS_WARN + assert "pip install" in check.remediation.lower() + + def test_entry_point_relay_fail_when_missing(self, monkeypatch): + def fake_which(name): + if name == "llm-relay": + return None + return "/fake/{}".format(name) + + monkeypatch.setattr("llm_relay.verify.install.shutil.which", fake_which) + report = verify_install() + check = next(c for c in report.checks if c.id == "entry_point_relay") + assert check.status == STATUS_FAIL + # The whole report should fail because of this + assert report.overall == STATUS_FAIL + + def test_proxy_extras_pass_when_all_importable(self): + # In the test env httpx/uvicorn/starlette are all installed. + report = verify_install() + check = next(c for c in report.checks if c.id == "proxy_extras") + assert check.status == STATUS_PASS + + def test_proxy_extras_warn_when_one_missing(self, monkeypatch): + import importlib as real_importlib + original_import = real_importlib.import_module + + def selective_import(name, *args, **kwargs): + if name == "starlette": + raise ImportError("simulated missing starlette") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr( + "llm_relay.verify.install.importlib.import_module", + selective_import, + ) + report = verify_install() + check = next(c for c in report.checks if c.id == "proxy_extras") + assert check.status == STATUS_WARN + assert "starlette" in check.data["missing"] diff --git a/tests/test_verify/test_integration.py b/tests/test_verify/test_integration.py new file mode 100644 index 0000000..d1106c8 --- /dev/null +++ b/tests/test_verify/test_integration.py @@ -0,0 +1,166 @@ +"""Tests for verify integration checks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from llm_relay.verify import ( + STATUS_FAIL, + STATUS_PASS, + STATUS_SKIPPED, + STATUS_WARN, +) +from llm_relay.verify.integration import ALL_CLIS, verify_integration + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + """Redirect Path.home() and shutil.which to a controlled state. + + Also strips ANTHROPIC_BASE_URL from the process env so settings.json is the + sole signal for the proxy_route check (otherwise the developer's real + shell env leaks into the test result). + """ + fake = tmp_path / "home" + fake.mkdir() + monkeypatch.setattr("llm_relay.verify.integration.Path.home", lambda: fake) + # Default: no binaries present. Tests override per-CLI. + monkeypatch.setattr("llm_relay.verify.integration.shutil.which", lambda name: None) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + return fake + + +def _make_claude_settings(home: Path, *, base_url=None, mcp_registered=False) -> Path: + claude_dir = home / ".claude" + claude_dir.mkdir(parents=True) + settings = {} + if base_url: + settings["env"] = {"ANTHROPIC_BASE_URL": base_url} + if mcp_registered: + settings["mcpServers"] = {"llm-relay": {"command": "llm-relay-mcp", "type": "stdio"}} + path = claude_dir / "settings.json" + path.write_text(json.dumps(settings)) + return path + + +class TestVerifyIntegrationDispatch: + def test_single_cli_target_is_cli_id(self, fake_home): + report = verify_integration("claude-code") + assert report.target == "claude-code" + + def test_all_target_aggregates(self, fake_home): + report = verify_integration("all") + assert report.target == "integration" + # All sub-reports' check ids namespaced by cli_id + prefixes = {c.id.split(".", 1)[0] for c in report.checks} + assert prefixes == {"claude-code", "openai-codex", "gemini-cli"} + + def test_none_treated_as_all(self, fake_home): + report = verify_integration(None) + assert report.target == "integration" + + def test_unknown_cli_raises(self, fake_home): + with pytest.raises(ValueError): + verify_integration("unknown-cli") + + +class TestClaudeCodeIntegration: + def test_binary_missing_skips_all(self, fake_home): + # shutil.which returns None by default in fake_home + report = verify_integration("claude-code") + # When binary is missing, every check is skipped (no fail) + statuses = {c.status for c in report.checks} + assert STATUS_FAIL not in statuses + assert STATUS_SKIPPED in statuses + + def test_binary_present_no_settings(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/claude" if name == "claude" else None, + ) + report = verify_integration("claude-code") + binary = next(c for c in report.checks if c.id == "binary") + settings = next(c for c in report.checks if c.id == "settings_present") + assert binary.status == STATUS_PASS + assert settings.status == STATUS_FAIL + + def test_full_happy_path(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/claude" if name == "claude" else None, + ) + _make_claude_settings( + fake_home, base_url="http://localhost:8083", mcp_registered=True, + ) + report = verify_integration("claude-code") + assert report.overall == STATUS_PASS + + def test_proxy_route_warn_when_not_local(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/claude" if name == "claude" else None, + ) + _make_claude_settings( + fake_home, base_url="https://api.anthropic.com", mcp_registered=True, + ) + report = verify_integration("claude-code") + proxy = next(c for c in report.checks if c.id == "proxy_route") + assert proxy.status == STATUS_WARN + + def test_mcp_not_registered_fails(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/claude" if name == "claude" else None, + ) + _make_claude_settings( + fake_home, base_url="http://localhost:8083", mcp_registered=False, + ) + report = verify_integration("claude-code") + mcp = next(c for c in report.checks if c.id == "mcp_server") + assert mcp.status == STATUS_FAIL + + +class TestCodexIntegration: + def test_codex_proxy_route_always_skipped(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/codex" if name == "codex" else None, + ) + report = verify_integration("openai-codex") + proxy = next(c for c in report.checks if c.id == "proxy_route") + assert proxy.status == STATUS_SKIPPED + # upstream limitation surfaced in data + assert proxy.data["reason"] == "upstream" + + +class TestGeminiIntegration: + def test_gemini_oauth_known_issue_is_warn(self, fake_home, monkeypatch): + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/gemini" if name == "gemini" else None, + ) + (fake_home / ".gemini").mkdir() + report = verify_integration("gemini-cli") + known = next(c for c in report.checks if c.id == "oauth_known_issue") + assert known.status == STATUS_WARN + assert "25425" in known.data["upstream_issue"] + + +class TestAggregatedAll: + def test_all_clis_present_aggregates(self, fake_home, monkeypatch): + # Make all three binaries appear present + monkeypatch.setattr( + "llm_relay.verify.integration.shutil.which", + lambda name: "/fake/{}".format(name) if name in {"claude", "codex", "gemini"} else None, + ) + _make_claude_settings(fake_home, base_url="http://localhost:8083", mcp_registered=True) + (fake_home / ".codex").mkdir() + (fake_home / ".codex" / "config.toml").write_text("# fake\n") + (fake_home / ".gemini").mkdir() + report = verify_integration(ALL_CLIS) + # All three CLI sub-reports show up, namespaced by id + prefixes = {c.id.split(".", 1)[0] for c in report.checks} + assert prefixes == {"claude-code", "openai-codex", "gemini-cli"}