feat(onboarding): add verify primitives (install/config/integration)

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to llm-relay are documented here.
 ## [Unreleased]
 
 ### Added
+- **`llm-relay verify` command group** (`verify/`): four idempotent verification subcommands designed for agent-driven onboarding. `verify install` confirms the package itself (Python version, importability, entry points, optional extras, version consistency). `verify config` confirms local state (db dir, schema, writability, config file, knowledge dir, port availability, no deprecated env vars). `verify integration --cli {claude-code,openai-codex,gemini-cli,all}` confirms each CLI is wired through the relay (binary on PATH, settings file present, `ANTHROPIC_BASE_URL` routing to localhost, MCP server registered). `verify all` aggregates install + config + integration. Shared output schema (`schema_version: "1"`) emits per-check status (`pass`/`fail`/`warn`/`skipped`) with optional remediation strings; overall priority is `fail > warn > pass`. Skipped checks (e.g. CLI not installed) never escalate to fail. Optional `--live` probes `/_health` on the proxy. Common options: `--format {text,json}`, `--quiet`, `--no-remediation`. Exit code is 0 on pass/warn, 1 on fail.
 - **`llm-relay env-fingerprint` command** (`env_fingerprint.py`): single-shot, idempotent snapshot of the local LLM CLI environment for agent-driven onboarding. Emits a structured JSON (or YAML) document with sections for `llm_relay` package state, per-CLI install/auth/version, port availability, filesystem layout, redacted environment variables (API keys reported as `set`/`empty`/`null` only), and an optional `doctor` summary. Designed so an agent (Claude Code / Codex / Gemini) automating an llm-relay install can parse the environment instead of scraping `init` output. Schema versioned (`schema_version: "1"`); sub-probe failures surface as `_error` markers without crashing the snapshot. Options: `--format {json,yaml}`, `--no-doctor`, `--ports`.
 
 ### Performance

src/llm_relay/detect/cli.py

@@ -278,6 +278,168 @@ def env_fingerprint(fmt: str, no_doctor: bool, ports_str: str) -> None:
         click.echo(json.dumps(snapshot, indent=2, ensure_ascii=False))
 
 
+# ── verify ──────────────────────────────────────────────────────────────────
+
+
+_VERIFY_FORMAT_OPTION = click.option(
+    "--format", "fmt",
+    type=click.Choice(["text", "json"]),
+    default="text",
+    help="Output format (default: text, json for agent consumption).",
+)
+_VERIFY_QUIET_OPTION = click.option(
+    "--quiet", "-q",
+    is_flag=True,
+    help="Suppress checks that passed; show only warn/fail/skipped.",
+)
+_VERIFY_NO_REMEDIATION_OPTION = click.option(
+    "--no-remediation",
+    is_flag=True,
+    help="Omit remediation hints from output.",
+)
+
+
+def _render_verify_report(report, fmt: str, quiet: bool, no_remediation: bool) -> int:
+    """Render a VerifyReport. Returns the desired exit code."""
+    import json as _json
+
+    data = report.to_dict()
+
+    if fmt == "json":
+        if no_remediation:
+            for c in data["checks"]:
+                c.pop("remediation", None)
+        if quiet:
+            data["checks"] = [c for c in data["checks"] if c["status"] != "pass"]
+        click.echo(_json.dumps(data, indent=2, ensure_ascii=False))
+    else:
+        status_styles = {"pass": "green", "fail": "red", "warn": "yellow", "skipped": "blue"}
+        click.echo("target:  {}".format(data["target"]))
+        click.echo("overall: {}".format(
+            click.style(data["overall"], fg=status_styles.get(data["overall"], ""))
+        ))
+        summary = data["summary"]
+        click.echo("summary: {}".format(
+            " ".join("{}={}".format(k, v) for k, v in summary.items())
+        ))
+        click.echo("")
+        for c in data["checks"]:
+            if quiet and c["status"] == "pass":
+                continue
+            status_str = click.style(
+                "[{:7s}]".format(c["status"]),
+                fg=status_styles.get(c["status"], ""),
+            )
+            click.echo("{} {}: {}".format(status_str, c["id"], c["detail"]))
+            if c.get("remediation") and not no_remediation:
+                click.echo("          → {}".format(c["remediation"]))
+        click.echo("")
+
+    # Exit code: 0 on pass/warn, 1 on fail.
+    return 0 if data["overall"] != "fail" else 1
+
+
+@cli.group("verify")
+def verify_group() -> None:
+    """Idempotent verification checks for install / config / integration.
+
+    Designed to be consumed by an agent automating an llm-relay install:
+    `--format json` emits a stable schema (see src/llm_relay/verify/__init__.py).
+
+    Exit code is 0 on pass/warn, 1 on fail.
+    """
+
+
+@verify_group.command("install")
+@_VERIFY_FORMAT_OPTION
+@_VERIFY_QUIET_OPTION
+@_VERIFY_NO_REMEDIATION_OPTION
+def verify_install_cmd(fmt: str, quiet: bool, no_remediation: bool) -> None:
+    """Verify the llm-relay package itself is correctly installed."""
+    from llm_relay.verify.install import verify_install
+
+    report = verify_install()
+    exit_code = _render_verify_report(report, fmt, quiet, no_remediation)
+    raise SystemExit(exit_code)
+
+
+@verify_group.command("config")
+@click.option("--port", default=8083, type=int, help="Proxy port to probe (default: 8083).")
+@_VERIFY_FORMAT_OPTION
+@_VERIFY_QUIET_OPTION
+@_VERIFY_NO_REMEDIATION_OPTION
+def verify_config_cmd(port: int, fmt: str, quiet: bool, no_remediation: bool) -> None:
+    """Verify the local llm-relay config (db, config file, port)."""
+    from llm_relay.verify.config import verify_config
+
+    report = verify_config(port=port)
+    exit_code = _render_verify_report(report, fmt, quiet, no_remediation)
+    raise SystemExit(exit_code)
+
+
+@verify_group.command("integration")
+@click.option(
+    "--cli", "cli_id",
+    type=click.Choice(["claude-code", "openai-codex", "gemini-cli", "all"]),
+    default="all",
+    help="Which CLI integration to verify (default: all).",
+)
+@click.option(
+    "--live",
+    is_flag=True,
+    help="Also probe /_health on the proxy port (requires running server).",
+)
+@click.option("--port", default=8083, type=int, help="Proxy port for --live (default: 8083).")
+@_VERIFY_FORMAT_OPTION
+@_VERIFY_QUIET_OPTION
+@_VERIFY_NO_REMEDIATION_OPTION
+def verify_integration_cmd(
+    cli_id: str,
+    live: bool,
+    port: int,
+    fmt: str,
+    quiet: bool,
+    no_remediation: bool,
+) -> None:
+    """Verify a CLI is wired through llm-relay (settings, proxy route, MCP)."""
+    from llm_relay.verify.integration import verify_integration
+
+    report = verify_integration(cli_id, live=live, port=port)
+    exit_code = _render_verify_report(report, fmt, quiet, no_remediation)
+    raise SystemExit(exit_code)
+
+
+@verify_group.command("all")
+@click.option("--port", default=8083, type=int, help="Proxy port (default: 8083).")
+@click.option("--live", is_flag=True, help="Include live proxy /_health probe.")
+@_VERIFY_FORMAT_OPTION
+@_VERIFY_QUIET_OPTION
+@_VERIFY_NO_REMEDIATION_OPTION
+def verify_all_cmd(
+    port: int,
+    live: bool,
+    fmt: str,
+    quiet: bool,
+    no_remediation: bool,
+) -> None:
+    """Run install + config + integration (all CLIs) and aggregate."""
+    from llm_relay.verify import aggregate
+    from llm_relay.verify.config import verify_config
+    from llm_relay.verify.install import verify_install
+    from llm_relay.verify.integration import verify_integration
+
+    combined = aggregate(
+        "all",
+        [
+            verify_install(),
+            verify_config(port=port),
+            verify_integration("all", live=live, port=port),
+        ],
+    )
+    exit_code = _render_verify_report(combined, fmt, quiet, no_remediation)
+    raise SystemExit(exit_code)
+
+
 @cli.command()
 @click.option("--host", default="0.0.0.0", help="Bind address.")
 @click.option("--port", "-p", default=8083, type=int, help="Listen port.")

src/llm_relay/verify/__init__.py

@@ -0,0 +1,181 @@
+"""Verify primitives -- idempotent checks for install / config / integration.
+
+Each primitive returns a `VerifyReport`: a structured pass/fail/warn/skipped
+record per check, designed to be consumed either by a human (`--format text`)
+or by an agent automating an llm-relay install (`--format json`).
+
+This package complements `env_fingerprint`: env-fingerprint describes the
+*state* of the user's environment, verify *asserts expectations* about that
+state. An agent typically uses env-fingerprint to plan, then verify to
+confirm each step it took.
+
+Shared output schema (schema_version "1"):
+
+  {
+    "schema_version": "1",
+    "target": "install" | "config" | "integration",
+    "captured_at": "...",
+    "overall": "pass" | "fail" | "warn",
+    "summary": {"pass": N, "fail": N, "warn": N, "skipped": N},
+    "checks": [
+      {
+        "id": "...",
+        "label": "...",
+        "status": "pass" | "fail" | "warn" | "skipped",
+        "detail": "...",
+        "remediation": "..." | null,
+        "data": {...} | null
+      }
+    ]
+  }
+
+Status semantics:
+  pass     -- expectation met
+  warn     -- expectation met but with a caveat the operator/agent should know
+  fail     -- expectation NOT met; remediation should fix it
+  skipped  -- check was not applicable (e.g. CLI not installed for integration)
+
+Overall priority: fail > warn > (pass | skipped).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+SCHEMA_VERSION = "1"
+
+STATUS_PASS = "pass"
+STATUS_FAIL = "fail"
+STATUS_WARN = "warn"
+STATUS_SKIPPED = "skipped"
+
+_ALL_STATUSES = (STATUS_PASS, STATUS_FAIL, STATUS_WARN, STATUS_SKIPPED)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+@dataclass
+class VerifyCheck:
+    """A single verification result.
+
+    `data` is optional structured detail that an agent may need (e.g. the
+    actual port number that was found in use). It's separate from `detail`
+    so machine consumers don't have to parse the human-friendly string.
+    """
+    id: str
+    label: str
+    status: str
+    detail: str
+    remediation: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None
+
+    def __post_init__(self) -> None:
+        if self.status not in _ALL_STATUSES:
+            raise ValueError(
+                "VerifyCheck.status must be one of {} (got {!r})".format(
+                    _ALL_STATUSES, self.status,
+                )
+            )
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "label": self.label,
+            "status": self.status,
+            "detail": self.detail,
+            "remediation": self.remediation,
+            "data": self.data,
+        }
+
+
+@dataclass
+class VerifyReport:
+    """Aggregate verification result for a single target."""
+    target: str
+    captured_at: str = field(default_factory=_now_iso)
+    schema_version: str = SCHEMA_VERSION
+    checks: List[VerifyCheck] = field(default_factory=list)
+
+    @property
+    def overall(self) -> str:
+        """fail > warn > pass. Skipped checks don't influence overall."""
+        statuses = {c.status for c in self.checks}
+        if STATUS_FAIL in statuses:
+            return STATUS_FAIL
+        if STATUS_WARN in statuses:
+            return STATUS_WARN
+        return STATUS_PASS
+
+    @property
+    def summary(self) -> Dict[str, int]:
+        counts = {status: 0 for status in _ALL_STATUSES}
+        for c in self.checks:
+            counts[c.status] += 1
+        return counts
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "target": self.target,
+            "captured_at": self.captured_at,
+            "overall": self.overall,
+            "summary": self.summary,
+            "checks": [c.to_dict() for c in self.checks],
+        }
+
+
+def run_check(
+    check_id: str,
+    label: str,
+    fn: Callable[[], VerifyCheck],
+    *,
+    fallback_remediation: Optional[str] = None,
+) -> VerifyCheck:
+    """Run a check function, capturing exceptions as fail status.
+
+    A check function returns a fully formed `VerifyCheck`. If the function
+    raises, we synthesize a fail result so a single broken probe never
+    crashes the whole report -- partial output beats no output.
+    """
+    try:
+        result = fn()
+    except Exception as exc:  # noqa: BLE001 -- intentional broad capture
+        return VerifyCheck(
+            id=check_id,
+            label=label,
+            status=STATUS_FAIL,
+            detail="{}: {}".format(type(exc).__name__, exc),
+            remediation=fallback_remediation,
+            data={"_error": True},
+        )
+    # Sanity: enforce id/label consistency in case the check function returns
+    # an unrelated VerifyCheck by mistake. The provided id/label always win.
+    result.id = check_id
+    result.label = label
+    return result
+
+
+def aggregate(target: str, reports: List[VerifyReport]) -> VerifyReport:
+    """Combine multiple sub-reports into one (e.g. for `verify all`).
+
+    The target string identifies the combined report (e.g. "all" or
+    "integration" when collapsing per-CLI sub-reports). Check IDs from
+    sub-reports are namespaced as `{sub_target}.{original_id}` to avoid
+    collisions across sub-reports.
+    """
+    combined = VerifyReport(target=target)
+    for sub in reports:
+        for check in sub.checks:
+            combined.checks.append(VerifyCheck(
+                id="{}.{}".format(sub.target, check.id),
+                label=check.label,
+                status=check.status,
+                detail=check.detail,
+                remediation=check.remediation,
+                data=check.data,
+            ))
+    return combined