From 9523a1c65a5eeabbd391875606ba9c8280714f66 Mon Sep 17 00:00:00 2001 From: ArkNill <48707894+ArkNill@users.noreply.github.com> Date: Wed, 20 May 2026 16:22:35 +0900 Subject: [PATCH 1/2] feat(onboarding): add env-fingerprint command for agent-driven setup Introduces the first deliverable of the LLM-driven onboarding path (Path B): a single, idempotent command that emits a structured snapshot of the user's local LLM CLI environment. Designed to be consumed by an agent (Claude Code / Codex / Gemini) that is automating an llm-relay install on the user's behalf. The agent reads the JSON output and decides which install/configure steps to take, rather than scraping the human-friendly `init` output. llm-relay env-fingerprint [--format json|yaml] [--no-doctor] [--ports 8080,8083] Output sections (schema_version "1"): - llm_relay: package version + on-disk paths - clis: per-CLI install/version/auth/config_dir (registry-keyed, so unknown ids never appear and missing CLIs are explicit) - ports: free/in_use per probed TCP port - filesystem: home, claude_home, projects_dir, session count, knowledge dir presence - env: relevant env vars; API keys redacted to set/empty/None so the output can be safely pasted into a bug report - doctor (optional): summary + per-check status from run_doctor The module is a pure collector that composes existing probes (setup_init._detect_clis, orch.discovery.discover_all, recover.doctor.run_doctor, detect.scanner) so it does not duplicate detection logic and stays in sync with what the relay actually does. Each sub-probe is wrapped in a safe-call so a single failure produces an `_error` marker on its section without crashing the whole snapshot -- partial data is more useful to an agent than no data. Tests cover: schema shape, doctor toggle, port selection, CLI registry coverage, API-key redaction, sub-probe failure isolation, ISO timestamp with timezone, schema contract. This is foundational for the upcoming ONBOARDING playbook + verify primitives that complete Path B. --- src/llm_relay/detect/cli.py | 52 ++++++++ src/llm_relay/env_fingerprint.py | 215 +++++++++++++++++++++++++++++++ tests/test_env_fingerprint.py | 145 +++++++++++++++++++++ 3 files changed, 412 insertions(+) create mode 100644 src/llm_relay/env_fingerprint.py create mode 100644 tests/test_env_fingerprint.py diff --git a/src/llm_relay/detect/cli.py b/src/llm_relay/detect/cli.py index 28997c1..0f9af1c 100644 --- a/src/llm_relay/detect/cli.py +++ b/src/llm_relay/detect/cli.py @@ -226,6 +226,58 @@ def doctor(fix: bool) -> None: click.echo(f" -> {r.recommendation}") +@cli.command("env-fingerprint") +@click.option( + "--format", "fmt", + type=click.Choice(["json", "yaml"]), + default="json", + help="Output format (default: json).", +) +@click.option( + "--no-doctor", "no_doctor", + is_flag=True, + help="Skip doctor health checks (faster, install/version probe only).", +) +@click.option( + "--ports", "ports_str", + default="", + help="Comma-separated TCP ports to probe (default: 8080,8083).", +) +def env_fingerprint(fmt: str, no_doctor: bool, ports_str: str) -> None: + """Print a structured snapshot of the local LLM CLI environment. + + Designed to be consumed by an agent (Claude Code / Codex / Gemini) + during automated llm-relay onboarding (Path B). Safe to run repeatedly -- + this command makes no changes to the user's environment. + + Schema is documented in src/llm_relay/env_fingerprint.py. + """ + import json + from typing import List, Optional + + from llm_relay.env_fingerprint import collect_fingerprint + + ports: Optional[List[int]] = None + if ports_str.strip(): + try: + ports = [int(p.strip()) for p in ports_str.split(",") if p.strip()] + except ValueError as exc: + raise click.BadParameter("--ports must be comma-separated integers") from exc + + snapshot = collect_fingerprint(include_doctor=not no_doctor, ports=ports) + + if fmt == "yaml": + try: + import yaml # type: ignore[import-not-found] + except ImportError as exc: + raise click.ClickException( + "PyYAML not installed. Install with `pip install pyyaml` or use --format=json." + ) from exc + click.echo(yaml.safe_dump(snapshot, sort_keys=False, allow_unicode=True)) + else: + click.echo(json.dumps(snapshot, indent=2, ensure_ascii=False)) + + @cli.command() @click.option("--host", default="0.0.0.0", help="Bind address.") @click.option("--port", "-p", default=8083, type=int, help="Listen port.") diff --git a/src/llm_relay/env_fingerprint.py b/src/llm_relay/env_fingerprint.py new file mode 100644 index 0000000..19f4015 --- /dev/null +++ b/src/llm_relay/env_fingerprint.py @@ -0,0 +1,215 @@ +"""Environment fingerprint -- single-shot JSON snapshot of the user's LLM CLI environment. + +Used by both onboarding paths: + + Path A (human-driven): `llm-relay env-fingerprint` shows the user what their + environment looks like before running `init`. + + Path B (LLM-driven, primary): an agent (Claude Code / Codex / Gemini) runs + `llm-relay env-fingerprint --json` and parses the structured output to + decide which install / configure steps to take, without having to scrape + the human-friendly `init` output. + +This module is a pure collector -- it composes the existing probes +(setup_init._detect_clis, orch.discovery, recover.doctor, detect.scanner) +into one stable JSON schema. It does NOT change anything in the user's +environment; calling it is safe to run repeatedly. +""" + +from __future__ import annotations + +import os +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Dict, List, Optional + +# Schema version is bumped when the output shape changes in a way that breaks +# agents parsing earlier outputs. Patch-level additions (new fields) do not +# require a bump; agents should ignore unknown fields. +SCHEMA_VERSION = "1" + +# Environment variables we surface (read-only) so agents can see what the +# user already configured without inspecting their shell rc files. +_RELEVANT_ENV_VARS = ( + "ANTHROPIC_BASE_URL", + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "GEMINI_API_KEY", + "LLM_RELAY_DB", + "LLM_RELAY_HISTORY", + "LLM_RELAY_LANG", + "LLM_TOKEN_CEILING", +) + +# Ports the relay typically wants. Agents use this to spot conflicts before +# they try to `init`. +_DEFAULT_PROXY_PORTS = (8080, 8083) + + +def _safe_call(fn, default): + """Run a probe function and swallow any exception, returning `default`. + + Fingerprint must never crash because of a sub-probe failure -- partial + data with an error marker is more useful to a calling agent than no data. + """ + try: + return fn() + except Exception as exc: # noqa: BLE001 -- the whole point is to capture any failure + return {"_error": "{}: {}".format(type(exc).__name__, exc), "_value": default} + + +def _llm_relay_version() -> Optional[str]: + try: + return version("llm-relay") + except PackageNotFoundError: + return None + + +def _llm_relay_section() -> Dict[str, Any]: + """Versions + on-disk paths the relay itself uses.""" + from llm_relay.setup_init import db_dir_for_env + + db_dir = db_dir_for_env() + return { + "version": _llm_relay_version(), + "db_dir": str(db_dir), + "db_path": str(db_dir / "usage.db"), + "config_path": str(db_dir / "config.json"), + "knowledge_dir": str(db_dir / "knowledge"), + } + + +def _clis_section() -> List[Dict[str, Any]]: + """Per-CLI install + auth + version + redacted proxy/config hints. + + Combines orch.discovery (which probes auth) with setup_init._detect_clis + (which surfaces config_dir). Output is keyed by cli_id so agents can match + against their own identity. + """ + from llm_relay.orch.discovery import discover_all + from llm_relay.setup_init import _detect_clis + + detect_by_id = {c["id"]: c for c in _detect_clis()} + out: List[Dict[str, Any]] = [] + for status in discover_all(): + detect = detect_by_id.get(status.cli_id, {}) + config_dir = detect.get("config_dir") + out.append({ + "id": status.cli_id, + "binary_name": status.binary_name, + "binary_path": status.binary_path, + "installed": status.installed, + "version": status.version, + "auth": { + "cli_authenticated": status.cli_authenticated, + "api_key_env": status.api_key_name, + "api_key_set": status.api_key_available, + "preferred": status.preferred_auth.value, + }, + "config_dir": config_dir, + "config_dir_exists": Path(config_dir).is_dir() if config_dir else False, + }) + return out + + +def _ports_section(ports: List[int]) -> Dict[str, str]: + from llm_relay.setup_init import _is_port_in_use + return {str(p): "in_use" if _is_port_in_use(p) else "free" for p in ports} + + +def _filesystem_section() -> Dict[str, Any]: + """Project / session locations the relay knows about. + + Counts are bounded and cheap to compute; an agent should follow up with + `llm-relay scan` for detail rather than trying to derive everything here. + """ + from llm_relay.detect.scanner import discover_sessions, find_claude_home, find_projects_dir + from llm_relay.setup_init import db_dir_for_env + + claude_home = find_claude_home() + projects_dir = find_projects_dir() + sessions = discover_sessions(projects_dir) if projects_dir.is_dir() else [] + db_dir = db_dir_for_env() + return { + "home": str(Path.home()), + "claude_home": str(claude_home) if claude_home.exists() else None, + "projects_dir": str(projects_dir) if projects_dir.is_dir() else None, + "session_count": len(sessions), + "knowledge_dir": str(db_dir / "knowledge"), + "knowledge_dir_exists": (db_dir / "knowledge").is_dir(), + "db_dir_exists": db_dir.is_dir(), + } + + +def _env_section() -> Dict[str, Any]: + """Surface relevant env vars. API keys are reported as set/unset only -- + never the actual value -- so the fingerprint can be safely pasted into a + bug report. + """ + out: Dict[str, Any] = {} + for name in _RELEVANT_ENV_VARS: + val = os.environ.get(name) + if val is None: + out[name] = None + continue + # Redact secret-shaped vars + if name.endswith("_API_KEY"): + out[name] = "set" if val else "empty" + else: + out[name] = val + return out + + +def _doctor_section() -> Dict[str, Any]: + """Run the existing read-only doctor checks and summarise. + + Agents typically don't need every check's full report inline; we return a + summary plus per-check status. Agents wanting detail call `llm-relay + doctor` separately. + """ + from llm_relay.recover.doctor import run_doctor + + report = run_doctor(fix=False) + checks_by_status: Dict[str, int] = {} + items = [] + for result in report.results: + checks_by_status[result.status] = checks_by_status.get(result.status, 0) + 1 + items.append({ + "name": result.name, + "status": result.status, + "detail": result.detail, + "recommendation": result.recommendation or None, + }) + return { + "totals": checks_by_status, + "checks": items, + } + + +def collect_fingerprint( + *, + include_doctor: bool = True, + ports: Optional[List[int]] = None, +) -> Dict[str, Any]: + """Return the full environment fingerprint as a dict. + + Set `include_doctor=False` to skip the doctor checks when the caller only + wants a fast install/version probe (~10x faster on a cold cache). + """ + if ports is None: + ports = list(_DEFAULT_PROXY_PORTS) + + snapshot: Dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "captured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "llm_relay": _safe_call(_llm_relay_section, {}), + "clis": _safe_call(_clis_section, []), + "ports": _safe_call(lambda: _ports_section(ports), {}), + "filesystem": _safe_call(_filesystem_section, {}), + "env": _safe_call(_env_section, {}), + } + if include_doctor: + snapshot["doctor"] = _safe_call(_doctor_section, {"totals": {}, "checks": []}) + return snapshot diff --git a/tests/test_env_fingerprint.py b/tests/test_env_fingerprint.py new file mode 100644 index 0000000..a156562 --- /dev/null +++ b/tests/test_env_fingerprint.py @@ -0,0 +1,145 @@ +"""Tests for env_fingerprint module.""" + +from __future__ import annotations + +from llm_relay.env_fingerprint import ( + _DEFAULT_PROXY_PORTS, + _RELEVANT_ENV_VARS, + SCHEMA_VERSION, + _safe_call, + collect_fingerprint, +) + + +class TestSafeCall: + def test_returns_value_on_success(self): + assert _safe_call(lambda: {"k": 1}, default={}) == {"k": 1} + + def test_swallows_exception_and_marks_error(self): + def boom(): + raise RuntimeError("kaboom") + + result = _safe_call(boom, default={}) + assert "_error" in result + assert "RuntimeError" in result["_error"] + assert "kaboom" in result["_error"] + assert result["_value"] == {} + + +class TestCollectFingerprint: + def test_basic_shape(self): + snap = collect_fingerprint(include_doctor=False) + assert snap["schema_version"] == SCHEMA_VERSION + assert "captured_at" in snap + assert "llm_relay" in snap + assert "clis" in snap + assert "ports" in snap + assert "filesystem" in snap + assert "env" in snap + assert "doctor" not in snap # disabled + + def test_include_doctor_adds_section(self): + snap = collect_fingerprint(include_doctor=True) + assert "doctor" in snap + assert "totals" in snap["doctor"] + assert "checks" in snap["doctor"] + assert isinstance(snap["doctor"]["checks"], list) + + def test_default_ports_probed(self): + snap = collect_fingerprint(include_doctor=False) + ports = snap["ports"] + for p in _DEFAULT_PROXY_PORTS: + assert str(p) in ports + assert ports[str(p)] in ("free", "in_use") + + def test_custom_ports(self): + snap = collect_fingerprint(include_doctor=False, ports=[59999]) + assert "59999" in snap["ports"] + assert "8083" not in snap["ports"] + + def test_clis_section_contains_known_ids_when_present(self): + snap = collect_fingerprint(include_doctor=False) + cli_ids = {c["id"] for c in snap["clis"]} + # discover_all always returns the full registry; ids should match the + # known set even when the binary is missing. + assert cli_ids == {"claude-code", "openai-codex", "gemini-cli"} + + def test_cli_entries_have_required_fields(self): + snap = collect_fingerprint(include_doctor=False) + for entry in snap["clis"]: + assert set(entry.keys()) >= { + "id", "binary_name", "binary_path", "installed", + "version", "auth", "config_dir", "config_dir_exists", + } + assert set(entry["auth"].keys()) >= { + "cli_authenticated", "api_key_env", "api_key_set", "preferred", + } + + def test_env_section_redacts_api_keys(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-shouldnotleak-1234567890abcdef") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://localhost:8083") + snap = collect_fingerprint(include_doctor=False) + # API key is redacted to a presence marker, never the value + assert snap["env"]["ANTHROPIC_API_KEY"] == "set" + assert "shouldnotleak" not in str(snap["env"]) + # Non-secret env vars pass through verbatim + assert snap["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:8083" + + def test_env_section_handles_unset_api_key(self, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + snap = collect_fingerprint(include_doctor=False) + assert snap["env"]["ANTHROPIC_API_KEY"] is None + + def test_env_section_marks_empty_api_key_as_empty(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "") + snap = collect_fingerprint(include_doctor=False) + assert snap["env"]["ANTHROPIC_API_KEY"] == "empty" + + def test_env_section_covers_relevant_vars(self): + snap = collect_fingerprint(include_doctor=False) + assert set(snap["env"].keys()) == set(_RELEVANT_ENV_VARS) + + def test_filesystem_section_shape(self): + snap = collect_fingerprint(include_doctor=False) + fs = snap["filesystem"] + assert "home" in fs + assert "knowledge_dir" in fs + assert "session_count" in fs + assert isinstance(fs["session_count"], int) + + def test_subprobe_failure_does_not_crash_collection(self, monkeypatch): + # Force the doctor probe to raise; collection should still produce a + # snapshot with an error marker on the doctor section only. + def boom(fix=False): + raise RuntimeError("doctor exploded") + + monkeypatch.setattr("llm_relay.recover.doctor.run_doctor", boom) + snap = collect_fingerprint(include_doctor=True) + assert "_error" in snap["doctor"] + assert "doctor exploded" in snap["doctor"]["_error"] + # Other sections remain intact + assert "clis" in snap + assert isinstance(snap["clis"], list) + + def test_captured_at_is_utc_iso(self): + snap = collect_fingerprint(include_doctor=False) + ts = snap["captured_at"] + # ISO 8601 with timezone offset; should parse with datetime + from datetime import datetime + parsed = datetime.fromisoformat(ts) + assert parsed.tzinfo is not None + + +class TestSchemaContract: + """Ensures the schema version is bumped intentionally when shape changes.""" + + def test_schema_version_is_string(self): + assert isinstance(SCHEMA_VERSION, str) + assert SCHEMA_VERSION # non-empty + + def test_relevant_env_vars_are_documented(self): + # If a new env var is added to _RELEVANT_ENV_VARS, this test does not + # gate it directly -- but it forces the test reader to be aware that + # adding env vars is part of the schema contract. + assert "ANTHROPIC_BASE_URL" in _RELEVANT_ENV_VARS + assert "LLM_RELAY_DB" in _RELEVANT_ENV_VARS From ff633fd297f76dee52c367f620405b022015d2c7 Mon Sep 17 00:00:00 2001 From: ArkNill <48707894+ArkNill@users.noreply.github.com> Date: Wed, 20 May 2026 16:28:16 +0900 Subject: [PATCH 2/2] docs(changelog): note env-fingerprint command under Unreleased ### Added Adds the env-fingerprint entry to CHANGELOG.md so the structured- snapshot command is documented alongside the other Unreleased changes. No code change. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc281e..dc2009f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to llm-relay are documented here. ## [Unreleased] +### Added +- **`llm-relay env-fingerprint` command** (`env_fingerprint.py`): single-shot, idempotent snapshot of the local LLM CLI environment for agent-driven onboarding. Emits a structured JSON (or YAML) document with sections for `llm_relay` package state, per-CLI install/auth/version, port availability, filesystem layout, redacted environment variables (API keys reported as `set`/`empty`/`null` only), and an optional `doctor` summary. Designed so an agent (Claude Code / Codex / Gemini) automating an llm-relay install can parse the environment instead of scraping `init` output. Schema versioned (`schema_version: "1"`); sub-probe failures surface as `_error` markers without crashing the snapshot. Options: `--format {json,yaml}`, `--no-doctor`, `--ports`. + ### Performance - **Incremental composition cache** (`composition.py`, #16): `analyze_session_composition` and `analyze_session_composition_per_turn` now fold delta turns into cached state instead of re-walking the full session on every new turn. Previously the cache invalidated whenever any new turn arrived, forcing an O(n²) replay; on long-running sessions this caused `/api/v1/turns` to balloon to tens of seconds and daemon RSS to climb above 10 GB within ~30 min of normal multi-agent traffic. Cache now keys on `session_id` alone with `(max_turn_processed, accumulated, totals, …)` state; new turns trigger a `WHERE turn_number > max_turn_processed` fetch. Compaction events (`storage_mode="full"`) reset accumulated state and re-absorb the snapshot. Exception during fold falls back to a full rebuild so an incremental error can't poison the cache.