diff --git a/CHANGELOG.md b/CHANGELOG.md index 0041c18..1133453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,36 @@ Every entry reports millisecond measurement error and a confusion matrix. See `d ## [Unreleased] +## [1.15.0] - 2026-07-22 + +### Added +- **`hotato prove`: one command, one release proof.** Composes the evidence + lanes you already ran into a single fail-closed verdict with a portable + receipt: `--contracts` re-scores contracts through contract verify's own + logic, `--suite` runs a conversation-test suite, `--before/--after` scores + the battery rollup through `hotato verify` (with its min-n refusal), and + `--gauntlet` runs the bundled stress suite. It adds no new scoring engine. + Overall is pass only when every activated lane passed; any failure fails the + proof, and a lane that could not support its claim is inconclusive and exits + non-zero, so CI never reads "could not tell" as green. Writes + `proof.json` (schema `hotato.proof.v1`) plus `proof.md`: verdicts, counts, + relative names, and sha256 digests only, with a content address that + regenerates identically under a pinned `SOURCE_DATE_EPOCH`. +- **`docs/LIFECYCLE.md`: the whole loop on one page.** Observe, catch, pin, + test, prove, and the production edge that feeds the next loop, with every + command mapped to its step. + +### Changed +- **The story is the loop.** README, the docs index, and `docs/COMPARE.md` are + restructured around one sentence: every production failure becomes a portable + test, every candidate runs against it, and every release carries evidence. + The comparison page now walks the lifecycle side by side with hosted + platforms and states the three structural differences (free at any scale, + verdicts reproducible enough to gate a build, data that stays yours). +- **Concrete positioning.** Every surface leads with "Local-first testing and + observability for AI agents"; the positioning-lockstep test enforces the new + line and retires the vague platform noun. + ## [1.14.0] - 2026-07-21 ### Changed diff --git a/CITATION.cff b/CITATION.cff index cd68db4..03a0d11 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,6 +1,6 @@ cff-version: 1.2.0 message: If you use hotato, please cite it as below. -title: "hotato: the local-first AI engineering platform" +title: "hotato: local-first testing and observability for AI agents" abstract: >- Trace, evaluate, test, and gate any AI agent on your machine, with nothing leaving it. Four planes over one install: observe (traces, tokens, cost, and @@ -15,7 +15,7 @@ type: software authors: - name: "Attention Labs, Inc." license: MIT -version: 1.14.0 +version: 1.15.0 date-released: "2026-07-21" repository-code: "https://github.com/attenlabs/hotato" url: "https://hotato.dev" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8907efd..6bb5614 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to hotato -Thanks for being here. hotato is the local-first AI engineering platform: it simulates, +Thanks for being here. hotato is local-first testing and observability for AI agents: it simulates, evaluates, reviews, and tracks calls across five dimensions (outcome, policy, conversation, speech, reliability), with the evidence behind every result. Deterministic checks stay separate from the model-judged rubric, and no output is diff --git a/README.md b/README.md index 43e079f..d34164b 100644 --- a/README.md +++ b/README.md @@ -10,17 +10,17 @@ # hotato -**The local-first AI engineering platform.** +**Local-first testing and observability for AI agents.** -Everything you reach for a hosted platform to do: trace, evaluate, test, and gate your LLM and voice agents, on your own machine. Free at any scale. Byte-reproducible. Nothing leaves it. +Your evals are green. Your agent still ships bugs they can't see: talk-over, dead air, a tool it swore it ran. hotato catches them on your machine and gates CI so they stay fixed. **[hotato.dev](https://hotato.dev)** -Hosted observability and eval platforms meter your traffic, keep your traces and prompts on their servers, and score your evals with a model, so the number drifts and cannot gate a build. hotato runs the same four jobs (tracing, evals, tests, and CI gates) on your own machine: free and MIT at any scale, byte-for-byte reproducible, and offline by default. +Free, open-source, and deterministic: the same call scores the same way every run, so you can gate a build on it. Your traces and prompts never leave your machine, and there's no per-seat or per-event bill as you scale. -**Catch your first failure in seconds. One command, no account:** +**Your first catch in seconds. One command, no account:** ```console $ uvx hotato start --demo @@ -28,38 +28,41 @@ Conversation failed: Agent did not yield; measured talk-over was 2.66 s. talk-over 2.66s the agent kept talking while the caller held the floor ``` -Your text eval read the words on that call and passed it. The timing failed. hotato scores what the transcript can't see, pins the catch as a CI contract, and reproduces the verdict byte for byte on every machine. It measures timing and say-do, not intent. +That transcript passed every text eval. The timing did not. hotato pins the catch as a CI contract that reproduces byte for byte, so a fixed bug stays fixed. It measures timing and say-do, not intent. -## What it does +## The loop -Four planes, one install, nothing leaves your machine. +Every production failure becomes a portable test, every candidate runs against it, and every release carries evidence. Five steps, one command each, nothing leaves your machine. | | | | | :-- | :-- | :-- | | **Observe** | traces, tokens, cost, and latency, from the OTel spans you already emit | `hotato observe report traces/` | -| **Evaluate** | deterministic assertions plus a separated local-judge lane, no blended score | `hotato assert run` | -| **Test** | simulate calls, stress-test turn-taking, pin any failure as a fixture | `hotato gauntlet` | -| **Gate** | content-addressed contracts fail the build on a regression, in CI | `hotato contract verify` | +| **Catch** | deterministic scoring finds what text evals miss: timing, say-do, policy | `hotato investigate call.wav` | +| **Pin** | your label turns the catch into a portable, content-addressed contract | `hotato investigate label STATE#1 --expect yield` | +| **Test** | simulate, stress, and drive candidates against everything you pinned | `hotato gauntlet` | +| **Prove** | every evidence lane composed into one fail-closed release proof, in CI | `hotato prove --contracts contracts/` | -Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. +Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. Every verdict carries its evidence across five dimensions (outcome, policy, conversation, speech, reliability), and production feeds the next loop: `hotato production export-regression` turns a live session back into a test. -Every verdict carries its evidence, scored across five dimensions: outcome, policy, conversation, speech, and reliability. +The whole loop, command by command: [`docs/LIFECYCLE.md`](docs/LIFECYCLE.md). ## Why it is different -Same four jobs a hosted platform runs. Three things it cannot offer. +Same loop a hosted platform runs. Three things it cannot offer. | | hotato | Hosted platforms | | :-- | :-- | :-- | -| Trace, evaluate, test, gate | yes | yes | +| Observe, catch, pin, test, prove | yes | yes | | Price at scale | free, MIT, any volume | metered per seat and per event | | Verdicts | byte-for-byte reproducible, gate a build | vary run to run | | Your traces and prompts | stay on your machine | live on their servers | | Runs in CI, offline | yes | needs their service | -## From a bad call to a CI gate +Full comparison: [`docs/COMPARE.md`](docs/COMPARE.md) -One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call: +## From a bad call to a release proof + +One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call, and every gate you have composes into one receipt: ```console $ hotato investigate ./call.wav @@ -69,11 +72,15 @@ $ hotato investigate ./call.wav $ hotato investigate label '.hotato/investigate-state.json#1' --expect yield created hotato contract: call-8s-yield -$ hotato contract verify contracts/ - [FAIL] call-8s-yield 0/1 contracts pass; exit_code=1 +$ hotato prove --contracts contracts/ +hotato prove: proof -- overall FAIL (exit 1) + lane verdict counts + contracts fail contracts=1 passed=0 failed=1 tampered=0 refused=0 +content_id: sha256:2959c905ee6c1030... +proof: .hotato/proofs/proof/proof.json ``` -A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. Same input, same verdict, byte for byte, on every machine. +A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. `hotato prove` composes every lane you have (contracts, suites, before/after batteries, the stress suite) into one fail-closed verdict with a content-addressed receipt: pass only when every lane passed, and "could not tell" is never green. ## Quickstart @@ -84,8 +91,8 @@ uvx hotato start --demo hotato investigate ./call.wav # 3. pin the caught moment as a regression contract hotato investigate label '.hotato/investigate-state.json#1' --expect yield -# 4. gate every pull request on it -hotato contract verify contracts/ +# 4. compose every gate into one release proof, on every pull request +hotato prove --contracts contracts/ ``` Keep it with `pipx install hotato`, drive it over MCP with `uvx --from "hotato[mcp]" hotato-mcp`, or walk the path in [`docs/GETTING-STARTED.md`](docs/GETTING-STARTED.md). @@ -102,10 +109,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: attenlabs/hotato@v1.14.0 + - uses: attenlabs/hotato@v1.15.0 with: contracts: contracts/ - hotato-version: 1.14.0 + hotato-version: 1.15.0 ``` Copy-paste workflow with a commit-SHA pin: [`docs/CI.md`](docs/CI.md). diff --git a/README.pypi.md b/README.pypi.md index c46b5e8..336627e 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -10,17 +10,17 @@ # hotato -**The local-first AI engineering platform.** +**Local-first testing and observability for AI agents.** -Everything you reach for a hosted platform to do: trace, evaluate, test, and gate your LLM and voice agents, on your own machine. Free at any scale. Byte-reproducible. Nothing leaves it. +Your evals are green. Your agent still ships bugs they can't see: talk-over, dead air, a tool it swore it ran. hotato catches them on your machine and gates CI so they stay fixed. **[hotato.dev](https://hotato.dev)** -Hosted observability and eval platforms meter your traffic, keep your traces and prompts on their servers, and score your evals with a model, so the number drifts and cannot gate a build. hotato runs the same four jobs (tracing, evals, tests, and CI gates) on your own machine: free and MIT at any scale, byte-for-byte reproducible, and offline by default. +Free, open-source, and deterministic: the same call scores the same way every run, so you can gate a build on it. Your traces and prompts never leave your machine, and there's no per-seat or per-event bill as you scale. -**Catch your first failure in seconds. One command, no account:** +**Your first catch in seconds. One command, no account:** ```console $ uvx hotato start --demo @@ -28,38 +28,41 @@ Conversation failed: Agent did not yield; measured talk-over was 2.66 s. talk-over 2.66s the agent kept talking while the caller held the floor ``` -Your text eval read the words on that call and passed it. The timing failed. hotato scores what the transcript can't see, pins the catch as a CI contract, and reproduces the verdict byte for byte on every machine. It measures timing and say-do, not intent. +That transcript passed every text eval. The timing did not. hotato pins the catch as a CI contract that reproduces byte for byte, so a fixed bug stays fixed. It measures timing and say-do, not intent. -## What it does +## The loop -Four planes, one install, nothing leaves your machine. +Every production failure becomes a portable test, every candidate runs against it, and every release carries evidence. Five steps, one command each, nothing leaves your machine. | | | | | :-- | :-- | :-- | | **Observe** | traces, tokens, cost, and latency, from the OTel spans you already emit | `hotato observe report traces/` | -| **Evaluate** | deterministic assertions plus a separated local-judge lane, no blended score | `hotato assert run` | -| **Test** | simulate calls, stress-test turn-taking, pin any failure as a fixture | `hotato gauntlet` | -| **Gate** | content-addressed contracts fail the build on a regression, in CI | `hotato contract verify` | +| **Catch** | deterministic scoring finds what text evals miss: timing, say-do, policy | `hotato investigate call.wav` | +| **Pin** | your label turns the catch into a portable, content-addressed contract | `hotato investigate label STATE#1 --expect yield` | +| **Test** | simulate, stress, and drive candidates against everything you pinned | `hotato gauntlet` | +| **Prove** | every evidence lane composed into one fail-closed release proof, in CI | `hotato prove --contracts contracts/` | -Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. +Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. Every verdict carries its evidence across five dimensions (outcome, policy, conversation, speech, reliability), and production feeds the next loop: `hotato production export-regression` turns a live session back into a test. -Every verdict carries its evidence, scored across five dimensions: outcome, policy, conversation, speech, and reliability. +The whole loop, command by command: [`docs/LIFECYCLE.md`](https://github.com/attenlabs/hotato/blob/main/docs/LIFECYCLE.md). ## Why it is different -Same four jobs a hosted platform runs. Three things it cannot offer. +Same loop a hosted platform runs. Three things it cannot offer. | | hotato | Hosted platforms | | :-- | :-- | :-- | -| Trace, evaluate, test, gate | yes | yes | +| Observe, catch, pin, test, prove | yes | yes | | Price at scale | free, MIT, any volume | metered per seat and per event | | Verdicts | byte-for-byte reproducible, gate a build | vary run to run | | Your traces and prompts | stay on your machine | live on their servers | | Runs in CI, offline | yes | needs their service | -## From a bad call to a CI gate +Full comparison: [`docs/COMPARE.md`](https://github.com/attenlabs/hotato/blob/main/docs/COMPARE.md) -One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call: +## From a bad call to a release proof + +One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call, and every gate you have composes into one receipt: ```console $ hotato investigate ./call.wav @@ -69,11 +72,15 @@ $ hotato investigate ./call.wav $ hotato investigate label '.hotato/investigate-state.json#1' --expect yield created hotato contract: call-8s-yield -$ hotato contract verify contracts/ - [FAIL] call-8s-yield 0/1 contracts pass; exit_code=1 +$ hotato prove --contracts contracts/ +hotato prove: proof -- overall FAIL (exit 1) + lane verdict counts + contracts fail contracts=1 passed=0 failed=1 tampered=0 refused=0 +content_id: sha256:2959c905ee6c1030... +proof: .hotato/proofs/proof/proof.json ``` -A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. Same input, same verdict, byte for byte, on every machine. +A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. `hotato prove` composes every lane you have (contracts, suites, before/after batteries, the stress suite) into one fail-closed verdict with a content-addressed receipt: pass only when every lane passed, and "could not tell" is never green. ## Quickstart @@ -84,8 +91,8 @@ uvx hotato start --demo hotato investigate ./call.wav # 3. pin the caught moment as a regression contract hotato investigate label '.hotato/investigate-state.json#1' --expect yield -# 4. gate every pull request on it -hotato contract verify contracts/ +# 4. compose every gate into one release proof, on every pull request +hotato prove --contracts contracts/ ``` Keep it with `pipx install hotato`, drive it over MCP with `uvx --from "hotato[mcp]" hotato-mcp`, or walk the path in [`docs/GETTING-STARTED.md`](https://github.com/attenlabs/hotato/blob/main/docs/GETTING-STARTED.md). @@ -102,10 +109,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: attenlabs/hotato@v1.14.0 + - uses: attenlabs/hotato@v1.15.0 with: contracts: contracts/ - hotato-version: 1.14.0 + hotato-version: 1.15.0 ``` Copy-paste workflow with a commit-SHA pin: [`docs/CI.md`](https://github.com/attenlabs/hotato/blob/main/docs/CI.md). diff --git a/atlas/records/addressed-backchannel-yielded.json b/atlas/records/addressed-backchannel-yielded.json index 7b700dd..4a5cc57 100644 --- a/atlas/records/addressed-backchannel-yielded.json +++ b/atlas/records/addressed-backchannel-yielded.json @@ -164,7 +164,7 @@ "provenance": { "hotato": { "name": "hotato", - "version": "1.14.0" + "version": "1.15.0" }, "inputs": [ { @@ -196,7 +196,7 @@ ], "source_result_digest": "sha256:7a691d5993389b47f2b43b7afbe7316bbb0d662e703fa0f8a82e39b93b052c08" }, - "record_id": "sha256:1203b251213d691d03f9ce42219e47f6ac1eea508b5299f87371fdef4ba54949", + "record_id": "sha256:dd3c63fcaeff31c7af4cecce0612aa524e554f7ae40d0c362f2d5ddef8491472", "reproduction": { "argv": [ "hotato", @@ -222,7 +222,7 @@ "version": "1.0" }, "reproduction_metadata": { - "hotato_version": "1.14.0", + "hotato_version": "1.15.0", "reviewer_principal": "hotato-examples", "working_directory": ".", "bundle": { @@ -240,7 +240,7 @@ "verify_output": "verify.json", "selector": "fd-02-backchannel-yielded", "record_out": "record", - "expected_record_id": "sha256:1203b251213d691d03f9ce42219e47f6ac1eea508b5299f87371fdef4ba54949" + "expected_record_id": "sha256:dd3c63fcaeff31c7af4cecce0612aa524e554f7ae40d0c362f2d5ddef8491472" }, "supersedes": { "record_id": "sha256:6c5e0e6de73004d8683c0987812d849c46fa9f9f8f204eb4da71e600b636e3a0", @@ -268,8 +268,8 @@ }, { "command": "hotato record render verify.json#fd-02-backchannel-yielded --out record", - "output": "wrote failure record (FAIL, fd-02-backchannel-yielded) to record/: failure-record.html, failure-record.json, failure-record.md, failure-record.svg\nrecord_id: sha256:1203b251213d691d03f9ce42219e47f6ac1eea508b5299f87371fdef4ba54949\n" + "output": "wrote failure record (FAIL, fd-02-backchannel-yielded) to record/: failure-record.html, failure-record.json, failure-record.md, failure-record.svg\nrecord_id: sha256:dd3c63fcaeff31c7af4cecce0612aa524e554f7ae40d0c362f2d5ddef8491472\n" } ], - "content_digest": "sha256:316481e041ab644927ebcd0512e0628772b4d6d5afe4295075a546d4003abd4a" + "content_digest": "sha256:dc05c98c555099225b575ebc5de7f8f95f9c94bcfd6e01345cf3f02f92c258ec" } diff --git a/atlas/records/addressed-interruption-missed.json b/atlas/records/addressed-interruption-missed.json index b97721e..a7e5fd1 100644 --- a/atlas/records/addressed-interruption-missed.json +++ b/atlas/records/addressed-interruption-missed.json @@ -164,7 +164,7 @@ "provenance": { "hotato": { "name": "hotato", - "version": "1.14.0" + "version": "1.15.0" }, "inputs": [ { @@ -196,7 +196,7 @@ ], "source_result_digest": "sha256:d3ec5aa5bbbdac05ef1ee7ad6d2612afd3916504e313e16df56ea3c334cb1346" }, - "record_id": "sha256:37f659dc05c614686403266a261131c5bf89e05b69be0750f42f1e92b3052208", + "record_id": "sha256:ad9c62d49f76f62c11c25184a13cd63669e5e6fee9f1dfc5f8093fb35dbe6eaa", "reproduction": { "argv": [ "hotato", @@ -222,7 +222,7 @@ "version": "1.0" }, "reproduction_metadata": { - "hotato_version": "1.14.0", + "hotato_version": "1.15.0", "reviewer_principal": "hotato-examples", "working_directory": ".", "bundle": { @@ -242,7 +242,7 @@ "verify_output": "verify.json", "selector": "fd-01-missed-interruption", "record_out": "record", - "expected_record_id": "sha256:37f659dc05c614686403266a261131c5bf89e05b69be0750f42f1e92b3052208" + "expected_record_id": "sha256:ad9c62d49f76f62c11c25184a13cd63669e5e6fee9f1dfc5f8093fb35dbe6eaa" }, "supersedes": { "record_id": "sha256:772fbb2a13addeb5cfbfb82d3c1652b15cb547d004e2db3db4341779c99c4336", @@ -270,8 +270,8 @@ }, { "command": "hotato record render verify.json#fd-01-missed-interruption --out record", - "output": "wrote failure record (FAIL, fd-01-missed-interruption) to record/: failure-record.html, failure-record.json, failure-record.md, failure-record.svg\nrecord_id: sha256:37f659dc05c614686403266a261131c5bf89e05b69be0750f42f1e92b3052208\n" + "output": "wrote failure record (FAIL, fd-01-missed-interruption) to record/: failure-record.html, failure-record.json, failure-record.md, failure-record.svg\nrecord_id: sha256:ad9c62d49f76f62c11c25184a13cd63669e5e6fee9f1dfc5f8093fb35dbe6eaa\n" } ], - "content_digest": "sha256:b7d4a6cfe5aabaa439a9ff27e2898cf6e805b0c6087febb4b14a8bc563176a48" + "content_digest": "sha256:c3d8d1ed551fc070610b61304a316e9166296a84128b23b963a533ad610f371e" } diff --git a/docs/CI.md b/docs/CI.md index 30e3b49..c983a3a 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -19,7 +19,7 @@ Conversation failed: Agent did not yield; measured talk-over was 2.66 s. Share in a PR: hotato-failure-record/failure-record.md Share as an image: hotato-failure-record/failure-record.svg - Verify the record: uvx --from hotato==1.14.0 hotato record verify hotato-failure-record/failure-record.json + Verify the record: uvx --from hotato==1.15.0 hotato record verify hotato-failure-record/failure-record.json ``` Preview it locally, then scaffold the durable gate into your own repository @@ -43,11 +43,11 @@ offline -- it runs the pinned Action revision itself off PYTHONPATH (no pip, no package index), installs no model, no ASR, no Node tool, calls no external judge, and reads no secret. -Composite Action since v1.4.0. Adopt the current release (v1.14.0), pinned +Composite Action since v1.4.0. Adopt the current release (v1.15.0), pinned by its full commit SHA; resolve the tag to its SHA first: ```bash -git ls-remote https://github.com/attenlabs/hotato refs/tags/v1.14.0 +git ls-remote https://github.com/attenlabs/hotato refs/tags/v1.15.0 ``` Then commit this workflow (replace the `attenlabs/hotato` pin with the SHA @@ -69,7 +69,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - id: hotato # Pin by full commit SHA (immutable); the comment names the release. - uses: attenlabs/hotato@ # v1.14.0 + uses: attenlabs/hotato@ # v1.15.0 with: suite: tests/voice/qa.suite.json agent: support-agent @@ -107,7 +107,7 @@ Everything else is optional: | Value | Effect | |---|---| | `action` (default) | Installs the pinned Action revision itself, `--no-deps`, no package-index egress -- exactly the revision your workflow pinned | -| an exact version, e.g. `1.14.0` | `pip install --no-deps hotato==1.14.0` | +| an exact version, e.g. `1.15.0` | `pip install --no-deps hotato==1.15.0` | | `preinstalled` | Skips installation (hotato is already on the runner) | A range or `latest` is refused, so the pin always names one exact diff --git a/docs/COMPARE.md b/docs/COMPARE.md index 1d4bb9a..ba7254f 100644 --- a/docs/COMPARE.md +++ b/docs/COMPARE.md @@ -1,92 +1,65 @@ -# Compare: Hotato vs broad QA platforms +# Compare: hotato vs hosted platforms -Hotato is an open-source, self-hosted conversation-QA system. This page -maps where it sits next to runtime layers and hosted platforms. +Local-first testing and observability for AI agents, next to the hosted +platforms that run the same jobs as a service. The short version: the same +lifecycle, three structural differences. -**Use Hamming, Cekura, Coval, Bluejay, Roark, Vapi, or Retell for:** broad -session QA, synthetic simulation, task success, transcript rubrics, -compliance workflows, production dashboards, load testing. +## The lifecycle, side by side -**Use Hotato for:** portable failure contracts from live calls, -local/private timing evidence, CI-enforced regression tests, trace-backed -turn-taking proof, refusing unsafe threshold bandaids. +Every step below runs on your machine, from the CLI, with an exit code CI can +gate on. The right column is what the equivalent step costs on a hosted +platform, structurally, whoever the vendor is. -**Hotato answers:** "Is the recorded evidence for this exact timing -failure still intact (every push), and does the CURRENT agent still avoid -it (on a fresh recapture, [`docs/RECAPTURE.md`](RECAPTURE.md))?" -**A broad QA platform answers:** "Was the whole call successful?" - -The two are complementary. A team running one of those platforms for -broad QA, or as the agent runtime itself, still needs the narrower answer: -is the specific talk-over or false-stop moment a caller hit last week -still fixed after today's prompt change? That is the layer Hotato owns. - -## What each platform is built for - -Capability descriptions only, sourced from each platform's own public -launch and product material. - -| Platform | Built for | -|---|---| -| **Hamming** | Prompt-to-test generation, simulated call batteries, production-replay CI/CD regression | -| **Cekura** | Production-call ingestion, test-case extraction, simulated scenarios, fleet monitoring, regulated verticals | -| **Coval** | Voice/chat agent simulation and evaluation, with published testing-approach comparisons | -| **Bluejay** | Synthetic stress-testing for voice agents at scale | -| **Roark** | Production-call replay QA, preserving what the caller said, how, and when | -| **Vapi** | Voice agent orchestration -- the runtime stack you build and deploy your agent on | -| **Retell** | Voice agent orchestration for building and deploying voice agents | - -## Three layers, three jobs - -Easy to confuse, so it's worth being exact about which one answers which -question: - -| Job | Best fit | Why | +| Lifecycle step | hotato | Hosted platforms | |---|---|---| -| Grade the whole call -- QA, transcript rubrics, task success, dashboards | Hamming, Cekura, Coval, Bluejay, Roark, Vapi, Retell | Grades the whole conversation and content, or is the agent platform itself | -| Catch an interruption live -- predict endpointing, suppress barge-in on noise, tune the turn detector | Pipecat, LiveKit (and similar runtime layers) | Acts *during* the call; Hotato runs *after*, from the recording | -| Prove one specific timing bug stays fixed, portably, byte-stable, on your machine | Hotato | A private, deterministic fixture -- audio, a human label, an explicit policy -- scored the same way everywhere with `hotato verify`; a fresh recapture ([`docs/RECAPTURE.md`](RECAPTURE.md)) checks whether the live agent still avoids it | - -Run every layer you need. A runtime layer improves the median call. Hotato -catches evidence and policy drift on every push and, recaptured, shows -whether a fix holds releases later: freeze the moment a runtime layer got -wrong with `hotato fixture promote`, fail CI when the frozen evidence -regresses, and recapture ([`docs/RECAPTURE.md`](RECAPTURE.md)) to check -the live agent. - -## What Hotato is for, precisely - -- **Private.** Scoring, scanning, reports, fixtures, and verification run - offline -- audio stays on your machine unless you explicitly pull it - from your own stack. See [THREAT-MODEL.md](THREAT-MODEL.md). -- **Deterministic under pinned inputs.** The reference backend is - rule-based end to end: the same hotato version, audio, channel map, - event onset, label, and scoring config always produce the same numbers - -- a changed result means a pinned input, policy, or scorer component - changed, most commonly the audio itself. Byte-identical re-runs are - verified in CI on Linux x86_64, Python 3.10, 3.11, and 3.12: - [VALIDATION.md](VALIDATION.md) Job 1. -- **Portable.** A confirmed failure becomes a labelled fixture (audio, - human label, explicit policy) that travels with the repository and - verifies the same way anywhere with `hotato verify`, deterministic for - a fixed hotato version. -- **Narrow on purpose.** Three timing signals, and only those: talking - over the caller, false-stopping on a backchannel, yielding too slowly. +| **Observe** traces, cost, latency | `hotato observe` on the OTel spans you already emit, locally | your traces live on their servers | +| **Catch** the failures evals miss | `hotato investigate` / `sweep`: deterministic timing and say-do scoring | model-scored, so the number drifts run to run | +| **Pin** a failure as a test | `hotato investigate label`: a portable, content-addressed contract | a dataset row inside their account | +| **Test** candidates against it | `hotato simulate` / `drive` / `gauntlet`, seeded and byte-reproducible | simulation credits, metered per run | +| **Prove** a release | `hotato prove`: every lane composed into one fail-closed receipt | a dashboard score you cannot re-derive | +| **Confirm** in production | `hotato production` alerts, exported back into the loop as tests | alerts inside their service | + +## The three structural differences + +- **Price at scale.** Free and MIT at any volume. There is no per-seat, per-run, + or per-event meter anywhere in the loop. +- **Verdicts you can gate on.** The same input scores the same way on every + machine, byte for byte, so an exit code can block a merge. A judged score + that varies run to run cannot. +- **Your data stays yours.** Recordings, traces, prompts, and backend state + never leave your machine, and a contract or proof stays verifiable if you + stop using hotato, if the vendor changes, or if the service is down. + +## The layer hotato is not + +A runtime voice layer (an orchestration platform's endpointing, a turn +detector, barge-in suppression) acts *during* the call. hotato runs *after*, +from the recording and the trace: it measures what happened, pins what must not +happen again, and proves a candidate against it. Run the runtime layer that +makes your median call good; run hotato so the failure a caller hit last week +stays fixed on every push. The frozen moment a runtime layer got wrong is +exactly what `hotato investigate label` pins and `hotato prove` re-verifies. + +## What the scoring claims, precisely + +- Timing and say-do, not intent: energy over time, tool spans, and post-call + state decide verdicts; no output claims emotion or meaning. +- Two channels or a timestamped transcript in; a mono or mixed export is + refused as NOT SCORABLE, never guessed at. +- Five dimensions (outcome, policy, conversation, speech, reliability), each + reported on its own evidence; there is no blended score anywhere. +- Deterministic checks stay separate from the model-judged rubric lane, which + is advisory and local. +- Determinism is verified in CI: byte-identical re-runs on Linux x86_64, + Python 3.10, 3.11, and 3.12 ([VALIDATION.md](VALIDATION.md) Job 1). ## Every example is scoped to one run -A Hotato example that shows a call failing on a named stack is labelled a +A hotato example that shows a call failing on a named stack is labelled a **provider-default** run: one assistant, one configuration, one date, one -scripted caller, on that provider's out-of-the-box interruption settings. -It demonstrates the threshold funnel -- a default config missing an -interruption and false-stopping on a backchannel in the same battery -- -scoped to that one run. Any stack, tuned, can pass the same fixtures. -Hotato publishes fixtures and reproduction steps: a record you can run -yourself, not a scoreboard. - -## The short version +scripted caller, on that provider's out-of-the-box interruption settings. It +demonstrates the threshold funnel scoped to that one run. Any stack, tuned, can +pass the same fixtures. hotato publishes fixtures and reproduction steps: a +record you can run yourself, not a scoreboard. -A QA platform grades the whole call. A runtime layer decides what the -agent does live. Hotato proves, privately and portably, that a specific -timing bug from a call is fixed, gates every push on the frozen evidence, -and recaptures to confirm it still holds on your current agent. +The full loop, command by command: [`docs/LIFECYCLE.md`](LIFECYCLE.md). diff --git a/docs/LIFECYCLE.md b/docs/LIFECYCLE.md new file mode 100644 index 0000000..e9a4923 --- /dev/null +++ b/docs/LIFECYCLE.md @@ -0,0 +1,131 @@ +# The hotato lifecycle + +Every production failure becomes a portable test, every candidate runs against +it, and every release carries evidence. That sentence is the whole product; this +page maps every command onto it. + +The loop has five steps you drive, and one that feeds itself: + +```text +OBSERVE -> CATCH -> PIN -> TEST -> PROVE + ^ | + +---------- production ----------- + +``` + +Eight nouns carry the loop: a **stack** (the platform your agent runs on), a +**recording** (one two-channel call), a **trace** (the call's pipeline events, +from your OTel spans), a **candidate** (a moment hotato flagged for judgment), +a **label** (your one-word verdict on it), a **contract** (the labeled moment, +packaged as a byte-reproducible check), a **suite** (a named set of tests run +together), and a **proof** (every evidence lane composed into one fail-closed +release verdict). + +## 1. Observe + +Your calls and traces, on your machine. Nothing leaves it. + +| What | Command | +| :-- | :-- | +| Store a stack credential once | `hotato connect --stack vapi` | +| Pull recent recordings in bulk | `hotato pull --stack vapi --since 7d` | +| Fetch and score one call | `hotato capture --stack vapi --call-id ID` | +| Capture OTel spans from any process | `hotato observe capture -- ` | +| Ingest an OTel export as a voice trace | `hotato trace ingest --otel spans.json` | +| Run the durable production evidence store | `hotato production serve` | +| Score a webhook's completed call | `hotato ingest --stack vapi --event call.json` | + +Cost, latency percentiles, and a self-contained dashboard come straight off the +captured traces: `hotato observe cost`, `hotato observe percentiles`, +`hotato observe report`. + +## 2. Catch + +Deterministic scoring finds what text evals miss: talk-over, dead air, slow +yields, a tool the agent claimed it ran. Five dimensions (outcome, policy, +conversation, speech, reliability), never one blended score, and every number +reproduces byte for byte. + +| What | Command | +| :-- | :-- | +| One recording, ranked candidate moments | `hotato investigate call.wav` | +| Every recent call on a stack, one sweep | `hotato sweep --stack vapi --since 7d` | +| A folder of recordings, ranked dashboard | `hotato analyze recordings/` | +| Is this recording scorable at all | `hotato trust --stereo call.wav` | +| Why a failing event failed, by layer | `hotato explain result.json` | +| Cluster failures across many runs | `hotato diagnose --fleet runs/` | + +A transcript works when there is no audio (`hotato investigate --transcript +t.json`), and a chat agent is driven directly (`hotato simulate scenario.json +--chat URL`). The scorer measures timing and say-do, not intent; a mono or +mixed export is refused as NOT SCORABLE, never guessed at. + +## 3. Pin + +You make the one human decision the loop needs: is this candidate a failure the +agent must never repeat? Your label turns it into a contract: clipped audio, +frame evidence, the policy it was judged under, and a content address, packaged +portable. It re-verifies on any machine, with or without hotato's help. + +| What | Command | +| :-- | :-- | +| Label a candidate, mint the contract | `hotato investigate label STATE#1 --expect yield` | +| Pin a moment from a sweep | `hotato fixture promote SWEEP.json#2 --expect hold` | +| File the confirm-or-ignore decision as a GitHub issue | `hotato issue create sweep.json` | +| Build a contract directly from a moment | `hotato contract create call.wav --onset 2.0 --expect yield` | +| Review and label at fleet scale | `hotato fleet review` / `hotato fleet label` | + +## 4. Test + +Run candidates against everything you have pinned, plus scripted callers, +personas, robustness batteries, and the bundled stress suite. Simulation is +deterministic: the same scenario and seed render the same bytes on every run. + +| What | Command | +| :-- | :-- | +| Render a scenario into a scored conversation | `hotato simulate scenario.json --out ./sim` | +| Drive a live call at a candidate agent | `hotato drive contracts/ID.hotato --stack vapi` | +| Stress one recording across noise and loss | `hotato battery robustness call.wav` | +| The bundled turn-taking stress suite | `hotato gauntlet` | +| A suite of conversation tests, per-dimension | `hotato suite run suite.json` | +| Before/after across a whole battery | `hotato verify --before old/ --after new/` | + +## 5. Prove + +One command composes every evidence lane you have into one fail-closed release +proof: contracts re-verified, suites re-run, before/after movement measured, +the stress suite cleared. The proof is a content-addressed receipt; CI gates on +the exit code, and the receipt stays verifiable anywhere. + +```bash +hotato prove --contracts contracts/ --before before/ --after after/ --out .hotato/proofs/v42/ +``` + +`pass` means every lane passed. Any failure fails the proof; a lane that could +not support its claim is `inconclusive`, and CI never reads "could not tell" as +green. The deeper machinery is there when you need it: `hotato fix trial` binds +a specific patch to its before/after evidence, `hotato apply --clone` stages a +candidate without touching production, and `hotato record render` turns any +failure into a share-safe card. + +## ...and production feeds the next loop + +The deploy is not the end of the loop; it is the next input. The production +evidence plane watches completed sessions, raises alerts on evidence gaps, and +exports any session as an offline-verifiable regression candidate, which lands +back in step 2. + +| What | Command | +| :-- | :-- | +| Continuous evidence intake | `hotato production serve` / `production ingest` | +| Alert transitions on session evidence | `hotato production alerts` | +| A session back into the loop as a test | `hotato production export-regression SESSION` | +| Canary a config change, roll it back | `hotato fleet canary start` / `rollback` | +| Team trends over releases | `hotato team` / `hotato release compare` | + +## Where the other commands sit + +`hotato start`, `demo`, `doctor`, and `init --auto` are onboarding: they walk +the loop on bundled calls before you spend a minute of your own. `hotato bench` +and `hotato gauntlet badge` prove the scorer itself. `hotato describe` emits +the machine-readable manifest a coding agent drives the loop with, and the MCP +server (`hotato-mcp`) exposes the same loop over stdio. diff --git a/docs/README.md b/docs/README.md index 83269fa..22ede35 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,10 +1,12 @@ # hotato docs -The local-first AI engineering platform: trace, evaluate, test, and gate any AI -agent on your machine. This index maps every doc to the plane it belongs to. +Local-first testing and observability for AI agents: every production failure +becomes a portable test, every candidate runs against it, and every release +carries evidence. This index maps every doc to the step it belongs to. New here? Start with **[GETTING-STARTED.md](GETTING-STARTED.md)**, then run -`uvx hotato start --demo`. +`uvx hotato start --demo`. The whole loop on one page: +**[LIFECYCLE.md](LIFECYCLE.md)**. ## Getting started diff --git a/docs/TRUST-GALLERY.md b/docs/TRUST-GALLERY.md index 786d0c5..cb28b97 100644 --- a/docs/TRUST-GALLERY.md +++ b/docs/TRUST-GALLERY.md @@ -1,7 +1,7 @@ # Trust gallery: eight recordings, eight verdicts Eight named input conditions, each with the `hotato` output it produces. -Every block below is verbatim CLI output from hotato 1.14.0, offline. The rows +Every block below is verbatim CLI output from hotato 1.15.0, offline. The rows correspond to the [trust matrix](TRUST-MATRIX.md). The clean and echo cases use the bundled examples diff --git a/llms-full.txt b/llms-full.txt index d034141..240354b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -14,17 +14,17 @@ FILE: README.md # hotato -**The local-first AI engineering platform.** +**Local-first testing and observability for AI agents.** -Everything you reach for a hosted platform to do: trace, evaluate, test, and gate your LLM and voice agents, on your own machine. Free at any scale. Byte-reproducible. Nothing leaves it. +Your evals are green. Your agent still ships bugs they can't see: talk-over, dead air, a tool it swore it ran. hotato catches them on your machine and gates CI so they stay fixed. **[hotato.dev](https://hotato.dev)** -Hosted observability and eval platforms meter your traffic, keep your traces and prompts on their servers, and score your evals with a model, so the number drifts and cannot gate a build. hotato runs the same four jobs (tracing, evals, tests, and CI gates) on your own machine: free and MIT at any scale, byte-for-byte reproducible, and offline by default. +Free, open-source, and deterministic: the same call scores the same way every run, so you can gate a build on it. Your traces and prompts never leave your machine, and there's no per-seat or per-event bill as you scale. -**Catch your first failure in seconds. One command, no account:** +**Your first catch in seconds. One command, no account:** ```console $ uvx hotato start --demo @@ -32,38 +32,41 @@ Conversation failed: Agent did not yield; measured talk-over was 2.66 s. talk-over 2.66s the agent kept talking while the caller held the floor ``` -Your text eval read the words on that call and passed it. The timing failed. hotato scores what the transcript can't see, pins the catch as a CI contract, and reproduces the verdict byte for byte on every machine. It measures timing and say-do, not intent. +That transcript passed every text eval. The timing did not. hotato pins the catch as a CI contract that reproduces byte for byte, so a fixed bug stays fixed. It measures timing and say-do, not intent. -## What it does +## The loop -Four planes, one install, nothing leaves your machine. +Every production failure becomes a portable test, every candidate runs against it, and every release carries evidence. Five steps, one command each, nothing leaves your machine. | | | | | :-- | :-- | :-- | | **Observe** | traces, tokens, cost, and latency, from the OTel spans you already emit | `hotato observe report traces/` | -| **Evaluate** | deterministic assertions plus a separated local-judge lane, no blended score | `hotato assert run` | -| **Test** | simulate calls, stress-test turn-taking, pin any failure as a fixture | `hotato gauntlet` | -| **Gate** | content-addressed contracts fail the build on a regression, in CI | `hotato contract verify` | +| **Catch** | deterministic scoring finds what text evals miss: timing, say-do, policy | `hotato investigate call.wav` | +| **Pin** | your label turns the catch into a portable, content-addressed contract | `hotato investigate label STATE#1 --expect yield` | +| **Test** | simulate, stress, and drive candidates against everything you pinned | `hotato gauntlet` | +| **Prove** | every evidence lane composed into one fail-closed release proof, in CI | `hotato prove --contracts contracts/` | -Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. +Deterministic. Byte-reproducible. Free, MIT. Agent-native over MCP. Every verdict carries its evidence across five dimensions (outcome, policy, conversation, speech, reliability), and production feeds the next loop: `hotato production export-regression` turns a live session back into a test. -Every verdict carries its evidence, scored across five dimensions: outcome, policy, conversation, speech, and reliability. +The whole loop, command by command: [`docs/LIFECYCLE.md`](docs/LIFECYCLE.md). ## Why it is different -Same four jobs a hosted platform runs. Three things it cannot offer. +Same loop a hosted platform runs. Three things it cannot offer. | | hotato | Hosted platforms | | :-- | :-- | :-- | -| Trace, evaluate, test, gate | yes | yes | +| Observe, catch, pin, test, prove | yes | yes | | Price at scale | free, MIT, any volume | metered per seat and per event | | Verdicts | byte-for-byte reproducible, gate a build | vary run to run | | Your traces and prompts | stay on your machine | live on their servers | | Runs in CI, offline | yes | needs their service | -## From a bad call to a CI gate +Full comparison: [`docs/COMPARE.md`](docs/COMPARE.md) + +## From a bad call to a release proof -One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call: +One recording in. The pinned failure becomes a gate that stays red until the agent stops failing that call, and every gate you have composes into one receipt: ```console $ hotato investigate ./call.wav @@ -73,11 +76,15 @@ $ hotato investigate ./call.wav $ hotato investigate label '.hotato/investigate-state.json#1' --expect yield created hotato contract: call-8s-yield -$ hotato contract verify contracts/ - [FAIL] call-8s-yield 0/1 contracts pass; exit_code=1 +$ hotato prove --contracts contracts/ +hotato prove: proof -- overall FAIL (exit 1) + lane verdict counts + contracts fail contracts=1 passed=0 failed=1 tampered=0 refused=0 +content_id: sha256:2959c905ee6c1030... +proof: .hotato/proofs/proof/proof.json ``` -A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. Same input, same verdict, byte for byte, on every machine. +A contract re-measures the captured failure under the pinned policy on every CI run, the same discipline a snapshot test gives you. `hotato prove` composes every lane you have (contracts, suites, before/after batteries, the stress suite) into one fail-closed verdict with a content-addressed receipt: pass only when every lane passed, and "could not tell" is never green. ## Quickstart @@ -88,8 +95,8 @@ uvx hotato start --demo hotato investigate ./call.wav # 3. pin the caught moment as a regression contract hotato investigate label '.hotato/investigate-state.json#1' --expect yield -# 4. gate every pull request on it -hotato contract verify contracts/ +# 4. compose every gate into one release proof, on every pull request +hotato prove --contracts contracts/ ``` Keep it with `pipx install hotato`, drive it over MCP with `uvx --from "hotato[mcp]" hotato-mcp`, or walk the path in [`docs/GETTING-STARTED.md`](docs/GETTING-STARTED.md). @@ -106,10 +113,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: attenlabs/hotato@v1.14.0 + - uses: attenlabs/hotato@v1.15.0 with: contracts: contracts/ - hotato-version: 1.14.0 + hotato-version: 1.15.0 ``` Copy-paste workflow with a commit-SHA pin: [`docs/CI.md`](docs/CI.md). @@ -6275,7 +6282,7 @@ Conversation failed: Agent did not yield; measured talk-over was 2.66 s. Share in a PR: hotato-failure-record/failure-record.md Share as an image: hotato-failure-record/failure-record.svg - Verify the record: uvx --from hotato==1.14.0 hotato record verify hotato-failure-record/failure-record.json + Verify the record: uvx --from hotato==1.15.0 hotato record verify hotato-failure-record/failure-record.json ``` Preview it locally, then scaffold the durable gate into your own repository @@ -6299,11 +6306,11 @@ offline -- it runs the pinned Action revision itself off PYTHONPATH (no pip, no package index), installs no model, no ASR, no Node tool, calls no external judge, and reads no secret. -Composite Action since v1.4.0. Adopt the current release (v1.14.0), pinned +Composite Action since v1.4.0. Adopt the current release (v1.15.0), pinned by its full commit SHA; resolve the tag to its SHA first: ```bash -git ls-remote https://github.com/attenlabs/hotato refs/tags/v1.14.0 +git ls-remote https://github.com/attenlabs/hotato refs/tags/v1.15.0 ``` Then commit this workflow (replace the `attenlabs/hotato` pin with the SHA @@ -6325,7 +6332,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - id: hotato # Pin by full commit SHA (immutable); the comment names the release. - uses: attenlabs/hotato@ # v1.14.0 + uses: attenlabs/hotato@ # v1.15.0 with: suite: tests/voice/qa.suite.json agent: support-agent @@ -6363,7 +6370,7 @@ Everything else is optional: | Value | Effect | |---|---| | `action` (default) | Installs the pinned Action revision itself, `--no-deps`, no package-index egress -- exactly the revision your workflow pinned | -| an exact version, e.g. `1.14.0` | `pip install --no-deps hotato==1.14.0` | +| an exact version, e.g. `1.15.0` | `pip install --no-deps hotato==1.15.0` | | `preinstalled` | Skips installation (hotato is already on the runner) | A range or `latest` is refused, so the pin always names one exact @@ -8620,98 +8627,71 @@ PYTHONPATH=src python3 scripts/render_card_assets.py FILE: docs/COMPARE.md ================================================================================ -# Compare: Hotato vs broad QA platforms - -Hotato is an open-source, self-hosted conversation-QA system. This page -maps where it sits next to runtime layers and hosted platforms. - -**Use Hamming, Cekura, Coval, Bluejay, Roark, Vapi, or Retell for:** broad -session QA, synthetic simulation, task success, transcript rubrics, -compliance workflows, production dashboards, load testing. +# Compare: hotato vs hosted platforms -**Use Hotato for:** portable failure contracts from live calls, -local/private timing evidence, CI-enforced regression tests, trace-backed -turn-taking proof, refusing unsafe threshold bandaids. - -**Hotato answers:** "Is the recorded evidence for this exact timing -failure still intact (every push), and does the CURRENT agent still avoid -it (on a fresh recapture, [`docs/RECAPTURE.md`](RECAPTURE.md))?" -**A broad QA platform answers:** "Was the whole call successful?" - -The two are complementary. A team running one of those platforms for -broad QA, or as the agent runtime itself, still needs the narrower answer: -is the specific talk-over or false-stop moment a caller hit last week -still fixed after today's prompt change? That is the layer Hotato owns. - -## What each platform is built for - -Capability descriptions only, sourced from each platform's own public -launch and product material. - -| Platform | Built for | -|---|---| -| **Hamming** | Prompt-to-test generation, simulated call batteries, production-replay CI/CD regression | -| **Cekura** | Production-call ingestion, test-case extraction, simulated scenarios, fleet monitoring, regulated verticals | -| **Coval** | Voice/chat agent simulation and evaluation, with published testing-approach comparisons | -| **Bluejay** | Synthetic stress-testing for voice agents at scale | -| **Roark** | Production-call replay QA, preserving what the caller said, how, and when | -| **Vapi** | Voice agent orchestration -- the runtime stack you build and deploy your agent on | -| **Retell** | Voice agent orchestration for building and deploying voice agents | +Local-first testing and observability for AI agents, next to the hosted +platforms that run the same jobs as a service. The short version: the same +lifecycle, three structural differences. -## Three layers, three jobs +## The lifecycle, side by side -Easy to confuse, so it's worth being exact about which one answers which -question: +Every step below runs on your machine, from the CLI, with an exit code CI can +gate on. The right column is what the equivalent step costs on a hosted +platform, structurally, whoever the vendor is. -| Job | Best fit | Why | +| Lifecycle step | hotato | Hosted platforms | |---|---|---| -| Grade the whole call -- QA, transcript rubrics, task success, dashboards | Hamming, Cekura, Coval, Bluejay, Roark, Vapi, Retell | Grades the whole conversation and content, or is the agent platform itself | -| Catch an interruption live -- predict endpointing, suppress barge-in on noise, tune the turn detector | Pipecat, LiveKit (and similar runtime layers) | Acts *during* the call; Hotato runs *after*, from the recording | -| Prove one specific timing bug stays fixed, portably, byte-stable, on your machine | Hotato | A private, deterministic fixture -- audio, a human label, an explicit policy -- scored the same way everywhere with `hotato verify`; a fresh recapture ([`docs/RECAPTURE.md`](RECAPTURE.md)) checks whether the live agent still avoids it | - -Run every layer you need. A runtime layer improves the median call. Hotato -catches evidence and policy drift on every push and, recaptured, shows -whether a fix holds releases later: freeze the moment a runtime layer got -wrong with `hotato fixture promote`, fail CI when the frozen evidence -regresses, and recapture ([`docs/RECAPTURE.md`](RECAPTURE.md)) to check -the live agent. - -## What Hotato is for, precisely - -- **Private.** Scoring, scanning, reports, fixtures, and verification run - offline -- audio stays on your machine unless you explicitly pull it - from your own stack. See [THREAT-MODEL.md](THREAT-MODEL.md). -- **Deterministic under pinned inputs.** The reference backend is - rule-based end to end: the same hotato version, audio, channel map, - event onset, label, and scoring config always produce the same numbers - -- a changed result means a pinned input, policy, or scorer component - changed, most commonly the audio itself. Byte-identical re-runs are - verified in CI on Linux x86_64, Python 3.10, 3.11, and 3.12: - [VALIDATION.md](VALIDATION.md) Job 1. -- **Portable.** A confirmed failure becomes a labelled fixture (audio, - human label, explicit policy) that travels with the repository and - verifies the same way anywhere with `hotato verify`, deterministic for - a fixed hotato version. -- **Narrow on purpose.** Three timing signals, and only those: talking - over the caller, false-stopping on a backchannel, yielding too slowly. +| **Observe** traces, cost, latency | `hotato observe` on the OTel spans you already emit, locally | your traces live on their servers | +| **Catch** the failures evals miss | `hotato investigate` / `sweep`: deterministic timing and say-do scoring | model-scored, so the number drifts run to run | +| **Pin** a failure as a test | `hotato investigate label`: a portable, content-addressed contract | a dataset row inside their account | +| **Test** candidates against it | `hotato simulate` / `drive` / `gauntlet`, seeded and byte-reproducible | simulation credits, metered per run | +| **Prove** a release | `hotato prove`: every lane composed into one fail-closed receipt | a dashboard score you cannot re-derive | +| **Confirm** in production | `hotato production` alerts, exported back into the loop as tests | alerts inside their service | + +## The three structural differences + +- **Price at scale.** Free and MIT at any volume. There is no per-seat, per-run, + or per-event meter anywhere in the loop. +- **Verdicts you can gate on.** The same input scores the same way on every + machine, byte for byte, so an exit code can block a merge. A judged score + that varies run to run cannot. +- **Your data stays yours.** Recordings, traces, prompts, and backend state + never leave your machine, and a contract or proof stays verifiable if you + stop using hotato, if the vendor changes, or if the service is down. + +## The layer hotato is not + +A runtime voice layer (an orchestration platform's endpointing, a turn +detector, barge-in suppression) acts *during* the call. hotato runs *after*, +from the recording and the trace: it measures what happened, pins what must not +happen again, and proves a candidate against it. Run the runtime layer that +makes your median call good; run hotato so the failure a caller hit last week +stays fixed on every push. The frozen moment a runtime layer got wrong is +exactly what `hotato investigate label` pins and `hotato prove` re-verifies. + +## What the scoring claims, precisely + +- Timing and say-do, not intent: energy over time, tool spans, and post-call + state decide verdicts; no output claims emotion or meaning. +- Two channels or a timestamped transcript in; a mono or mixed export is + refused as NOT SCORABLE, never guessed at. +- Five dimensions (outcome, policy, conversation, speech, reliability), each + reported on its own evidence; there is no blended score anywhere. +- Deterministic checks stay separate from the model-judged rubric lane, which + is advisory and local. +- Determinism is verified in CI: byte-identical re-runs on Linux x86_64, + Python 3.10, 3.11, and 3.12 ([VALIDATION.md](VALIDATION.md) Job 1). ## Every example is scoped to one run -A Hotato example that shows a call failing on a named stack is labelled a +A hotato example that shows a call failing on a named stack is labelled a **provider-default** run: one assistant, one configuration, one date, one -scripted caller, on that provider's out-of-the-box interruption settings. -It demonstrates the threshold funnel -- a default config missing an -interruption and false-stopping on a backchannel in the same battery -- -scoped to that one run. Any stack, tuned, can pass the same fixtures. -Hotato publishes fixtures and reproduction steps: a record you can run -yourself, not a scoreboard. +scripted caller, on that provider's out-of-the-box interruption settings. It +demonstrates the threshold funnel scoped to that one run. Any stack, tuned, can +pass the same fixtures. hotato publishes fixtures and reproduction steps: a +record you can run yourself, not a scoreboard. -## The short version - -A QA platform grades the whole call. A runtime layer decides what the -agent does live. Hotato proves, privately and portably, that a specific -timing bug from a call is fixed, gates every push on the frozen evidence, -and recaptures to confirm it still holds on your current agent. +The full loop, command by command: [`docs/LIFECYCLE.md`](LIFECYCLE.md). ================================================================================ @@ -9801,6 +9781,143 @@ and unobservable operations as such. Do not infer carrier behavior from a successful local method call. +================================================================================ +FILE: docs/LIFECYCLE.md +================================================================================ + +# The hotato lifecycle + +Every production failure becomes a portable test, every candidate runs against +it, and every release carries evidence. That sentence is the whole product; this +page maps every command onto it. + +The loop has five steps you drive, and one that feeds itself: + +```text +OBSERVE -> CATCH -> PIN -> TEST -> PROVE + ^ | + +---------- production ----------- + +``` + +Eight nouns carry the loop: a **stack** (the platform your agent runs on), a +**recording** (one two-channel call), a **trace** (the call's pipeline events, +from your OTel spans), a **candidate** (a moment hotato flagged for judgment), +a **label** (your one-word verdict on it), a **contract** (the labeled moment, +packaged as a byte-reproducible check), a **suite** (a named set of tests run +together), and a **proof** (every evidence lane composed into one fail-closed +release verdict). + +## 1. Observe + +Your calls and traces, on your machine. Nothing leaves it. + +| What | Command | +| :-- | :-- | +| Store a stack credential once | `hotato connect --stack vapi` | +| Pull recent recordings in bulk | `hotato pull --stack vapi --since 7d` | +| Fetch and score one call | `hotato capture --stack vapi --call-id ID` | +| Capture OTel spans from any process | `hotato observe capture -- ` | +| Ingest an OTel export as a voice trace | `hotato trace ingest --otel spans.json` | +| Run the durable production evidence store | `hotato production serve` | +| Score a webhook's completed call | `hotato ingest --stack vapi --event call.json` | + +Cost, latency percentiles, and a self-contained dashboard come straight off the +captured traces: `hotato observe cost`, `hotato observe percentiles`, +`hotato observe report`. + +## 2. Catch + +Deterministic scoring finds what text evals miss: talk-over, dead air, slow +yields, a tool the agent claimed it ran. Five dimensions (outcome, policy, +conversation, speech, reliability), never one blended score, and every number +reproduces byte for byte. + +| What | Command | +| :-- | :-- | +| One recording, ranked candidate moments | `hotato investigate call.wav` | +| Every recent call on a stack, one sweep | `hotato sweep --stack vapi --since 7d` | +| A folder of recordings, ranked dashboard | `hotato analyze recordings/` | +| Is this recording scorable at all | `hotato trust --stereo call.wav` | +| Why a failing event failed, by layer | `hotato explain result.json` | +| Cluster failures across many runs | `hotato diagnose --fleet runs/` | + +A transcript works when there is no audio (`hotato investigate --transcript +t.json`), and a chat agent is driven directly (`hotato simulate scenario.json +--chat URL`). The scorer measures timing and say-do, not intent; a mono or +mixed export is refused as NOT SCORABLE, never guessed at. + +## 3. Pin + +You make the one human decision the loop needs: is this candidate a failure the +agent must never repeat? Your label turns it into a contract: clipped audio, +frame evidence, the policy it was judged under, and a content address, packaged +portable. It re-verifies on any machine, with or without hotato's help. + +| What | Command | +| :-- | :-- | +| Label a candidate, mint the contract | `hotato investigate label STATE#1 --expect yield` | +| Pin a moment from a sweep | `hotato fixture promote SWEEP.json#2 --expect hold` | +| File the confirm-or-ignore decision as a GitHub issue | `hotato issue create sweep.json` | +| Build a contract directly from a moment | `hotato contract create call.wav --onset 2.0 --expect yield` | +| Review and label at fleet scale | `hotato fleet review` / `hotato fleet label` | + +## 4. Test + +Run candidates against everything you have pinned, plus scripted callers, +personas, robustness batteries, and the bundled stress suite. Simulation is +deterministic: the same scenario and seed render the same bytes on every run. + +| What | Command | +| :-- | :-- | +| Render a scenario into a scored conversation | `hotato simulate scenario.json --out ./sim` | +| Drive a live call at a candidate agent | `hotato drive contracts/ID.hotato --stack vapi` | +| Stress one recording across noise and loss | `hotato battery robustness call.wav` | +| The bundled turn-taking stress suite | `hotato gauntlet` | +| A suite of conversation tests, per-dimension | `hotato suite run suite.json` | +| Before/after across a whole battery | `hotato verify --before old/ --after new/` | + +## 5. Prove + +One command composes every evidence lane you have into one fail-closed release +proof: contracts re-verified, suites re-run, before/after movement measured, +the stress suite cleared. The proof is a content-addressed receipt; CI gates on +the exit code, and the receipt stays verifiable anywhere. + +```bash +hotato prove --contracts contracts/ --before before/ --after after/ --out .hotato/proofs/v42/ +``` + +`pass` means every lane passed. Any failure fails the proof; a lane that could +not support its claim is `inconclusive`, and CI never reads "could not tell" as +green. The deeper machinery is there when you need it: `hotato fix trial` binds +a specific patch to its before/after evidence, `hotato apply --clone` stages a +candidate without touching production, and `hotato record render` turns any +failure into a share-safe card. + +## ...and production feeds the next loop + +The deploy is not the end of the loop; it is the next input. The production +evidence plane watches completed sessions, raises alerts on evidence gaps, and +exports any session as an offline-verifiable regression candidate, which lands +back in step 2. + +| What | Command | +| :-- | :-- | +| Continuous evidence intake | `hotato production serve` / `production ingest` | +| Alert transitions on session evidence | `hotato production alerts` | +| A session back into the loop as a test | `hotato production export-regression SESSION` | +| Canary a config change, roll it back | `hotato fleet canary start` / `rollback` | +| Team trends over releases | `hotato team` / `hotato release compare` | + +## Where the other commands sit + +`hotato start`, `demo`, `doctor`, and `init --auto` are onboarding: they walk +the loop on bundled calls before you spend a minute of your own. `hotato bench` +and `hotato gauntlet badge` prove the scorer itself. `hotato describe` emits +the machine-readable manifest a coding agent drives the loop with, and the MCP +server (`hotato-mcp`) exposes the same loop over stdio. + + ================================================================================ FILE: docs/LIVEKIT-CALLER-SESSION.md ================================================================================ @@ -10785,11 +10902,13 @@ FILE: docs/README.md # hotato docs -The local-first AI engineering platform: trace, evaluate, test, and gate any AI -agent on your machine. This index maps every doc to the plane it belongs to. +Local-first testing and observability for AI agents: every production failure +becomes a portable test, every candidate runs against it, and every release +carries evidence. This index maps every doc to the step it belongs to. New here? Start with **[GETTING-STARTED.md](GETTING-STARTED.md)**, then run -`uvx hotato start --demo`. +`uvx hotato start --demo`. The whole loop on one page: +**[LIFECYCLE.md](LIFECYCLE.md)**. ## Getting started @@ -13011,7 +13130,7 @@ FILE: docs/TRUST-GALLERY.md # Trust gallery: eight recordings, eight verdicts Eight named input conditions, each with the `hotato` output it produces. -Every block below is verbatim CLI output from hotato 1.14.0, offline. The rows +Every block below is verbatim CLI output from hotato 1.15.0, offline. The rows correspond to the [trust matrix](TRUST-MATRIX.md). The clean and echo cases use the bundled examples diff --git a/llms.txt b/llms.txt index e8649a7..dbffa08 100644 --- a/llms.txt +++ b/llms.txt @@ -1,14 +1,15 @@ # hotato -> The local-first AI engineering platform: everything you use a hosted +> Local-first testing and observability for AI agents: everything you use a hosted > observability or eval platform for, running on your own machine. Trace, > evaluate, test, and gate your LLM and voice agents: free and MIT at any scale, > byte-reproducible so a verdict can gate a build, and offline by default with -> nothing leaving your machine. Four planes over one install -- observe -> (traces, tokens, cost, and latency from your OTel spans), evaluate -> (deterministic assertions plus a separated local-judge lane), test (simulate, -> the gauntlet stress suite, robustness), and gate (content-addressed CI -> contracts). The wedge leads the demo: give it a two-channel recording or a +> nothing leaving your machine. One loop over one install: observe (traces, +> tokens, cost, and latency from your OTel spans), catch (deterministic scoring +> finds what text evals miss), pin (a labeled failure becomes a portable +> content-addressed contract), test (simulate, drive, and the bundled stress +> suite), and prove (hotato prove composes every evidence lane into one +> fail-closed release proof). The wedge leads the demo: give it a two-channel recording or a > timestamped transcript and it measures turn timing and say-do evidence -- did > the agent yield when the caller took the floor, how fast, how many seconds > both talked at once, and did what the agent said match what the backend did. @@ -35,7 +36,7 @@ > --format json` regenerates the same command data straight from the CLI's > argparse, so it never drifts from the real flags. > -> Version 1.14.0 +> Version 1.15.0 ## Start here diff --git a/pyproject.toml b/pyproject.toml index adf4640..a937d57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,8 @@ build-backend = "setuptools.build_meta" [project] name = "hotato" -version = "1.14.0" -description = "The local-first AI engineering platform: trace, evaluate, test, and gate your LLM and voice agents on your own machine. Everything you use a hosted platform for, free and MIT at any scale, byte-reproducible, and CI-native. Nothing leaves your machine." +version = "1.15.0" +description = "Local-first testing and observability for AI agents: trace, evaluate, test, and gate your LLM and voice agents on your own machine. Everything you use a hosted platform for, free and MIT at any scale, byte-reproducible, and CI-native. Nothing leaves your machine." readme = "README.pypi.md" # Support policy: requires-python is lower-bounded only. There is no upper cap # on purpose -- an upper pin (e.g. <3.14) locks installs out of new interpreters diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index c76ccdb..ac928ac 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -1,6 +1,6 @@ # @hotato/sdk -Typed TypeScript client for [hotato](https://github.com/attenlabs/hotato), the local-first AI engineering platform. Each call spawns the `hotato` CLI with `--format json` and returns its document parsed into strict interfaces. The CLI's JSON output is the contract: one schema across your terminal, your CI logs, and this SDK. Scoring, reduction, and verification run inside the CLI; the SDK types what it says. +Typed TypeScript client for [hotato](https://github.com/attenlabs/hotato), local-first testing and observability for AI agents. Each call spawns the `hotato` CLI with `--format json` and returns its document parsed into strict interfaces. The CLI's JSON output is the contract: one schema across your terminal, your CI logs, and this SDK. Scoring, reduction, and verification run inside the CLI; the SDK types what it says. ## Install diff --git a/server.json b/server.json index 732ae28..ef8fdde 100644 --- a/server.json +++ b/server.json @@ -1,8 +1,8 @@ { "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.attenlabs/hotato", - "description": "The local-first AI engineering platform. Trace, evaluate, test, and gate any AI agent. MIT.", - "version": "1.14.0", + "description": "Local-first testing and observability for AI agents. Trace, evaluate, test, and gate. MIT.", + "version": "1.15.0", "repository": { "url": "https://github.com/attenlabs/hotato", "source": "github", @@ -14,7 +14,7 @@ "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "hotato", - "version": "1.14.0", + "version": "1.15.0", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/hotato/__init__.py b/src/hotato/__init__.py index 22c9ee1..b43f07e 100644 --- a/src/hotato/__init__.py +++ b/src/hotato/__init__.py @@ -1,4 +1,4 @@ -"""hotato: the local-first AI engineering platform. +"""hotato: local-first testing and observability for AI agents. Trace, evaluate, test, and gate any AI agent on your machine: four planes (observe, evaluate, test, gate) over one install. Turn-timing and say-do are @@ -64,6 +64,6 @@ # absent or can describe a DIFFERENT installed copy than the code executing. # tests/test_version_lockstep.py enforces the match; 0.4.0 shipped # self-reporting 0.3.1 because nothing did. -__version__ = "1.14.0" +__version__ = "1.15.0" __all__ = ["run_single", "run_suite", "LIMITS", "SUITE_ID", "__version__"] diff --git a/src/hotato/cli.py b/src/hotato/cli.py index 40ff552..261c6c7 100644 --- a/src/hotato/cli.py +++ b/src/hotato/cli.py @@ -612,6 +612,15 @@ def _atomic_write_json(path: str, obj) -> None: "distinct code, shared with hotato apply, lets a script tell it " "apart from a usage error"), ), + "prove": ( + (0, "every activated evidence lane passed (contracts, suite, " + "before/after, gauntlet -- whichever you activated)"), + (1, "fail-closed: at least one lane failed or regressed"), + (2, "usage error (zero lanes activated, an incomplete flag pair, " + "unusable input) or INCONCLUSIVE (a lane refused or came back " + "inconclusive and none failed) -- 'could not tell' is never " + "green"), + ), "loop": ( (0, "advanced the loop and persisted state (or re-reported where it " "left off)"), @@ -2114,6 +2123,31 @@ def _cmd_fix_trial(args) -> int: return result["exit_code"] +def _cmd_prove(args) -> int: + from . import prove as _prove + + # run_prove raises ValueError (exit 2) BEFORE anything runs or is written + # when zero lanes are activated or a flag pair is incomplete. + proof = _prove.run_prove( + contracts=args.contracts, suite=args.suite, agent=args.agent, + before=args.before, after=args.after, min_n=args.min_n, + gauntlet=args.gauntlet, name=args.name, + ) + out_dir = args.out or os.path.join(".hotato", "proofs", proof["name"]) + os.makedirs(out_dir, exist_ok=True) + json_path = os.path.join(out_dir, "proof.json") + md_path = os.path.join(out_dir, "proof.md") + _atomic_write_text(json_path, _prove.serialize(proof)) + _atomic_write_text(md_path, _prove.render_md(proof)) + if args.format == "json": + print(_errors.safe_json_dumps(proof, indent=2)) + print(f"wrote {json_path}", file=sys.stderr) + print(f"wrote {md_path}", file=sys.stderr) + else: + print(_prove.render_text(proof, proof_path=json_path), end="") + return proof["exit_code"] + + def _cmd_loop(args) -> int: from . import loop as _loop @@ -5142,7 +5176,7 @@ def build_parser() -> argparse.ArgumentParser: description=( _get_started_block() + "\n\n" - "Hotato: the local-first AI engineering platform. Trace, evaluate, " + "Hotato: local-first testing and observability for AI agents. Trace, evaluate, " "test, and gate any AI agent on your machine, with nothing leaving " "it. Scores turn timing and say-do across five dimensions (outcome, " "policy, conversation, speech, reliability) with the evidence behind " @@ -8939,6 +8973,83 @@ def build_parser() -> argparse.ArgumentParser: "and the attribution section)") ft.set_defaults(func=_cmd_fix_trial) + # --- prove: compose the evidence lanes into one release proof ----------- + pv = sub.add_parser( + "prove", + help="compose every evidence lane you have (contracts, suite, " + "before/after, gauntlet) into one fail-closed, content-addressed " + "release proof; exit 0 only when every lane passed", + description=( + "hotato prove composes the evidence lanes you already ran into " + "ONE portable release proof. Each flag activates one lane: " + "--contracts re-scores a directory of hotato contracts through " + "contract verify's own logic; --suite runs a suite.v1 of " + "conversation-tests through the suite runner; --before/--after " + "scores the battery rollup through hotato verify (with its " + "--min-n refusal); --gauntlet runs the bundled deterministic " + "stress suite. It adds no new scoring engine: every number here " + "is one contract verify / suite run / verify / gauntlet already " + "measures. The verdict is FAIL-CLOSED: overall pass only when " + "EVERY activated lane passed; any failed or regressed lane is a " + "fail; a refused or inconclusive lane (with none failed) is " + "inconclusive and exits non-zero, so CI never reads 'could not " + "tell' as green. Zero activated lanes is a usage error: a proof " + "of nothing is refused, not an empty pass. It writes " + "proof.json (schema hotato.proof.v1) + proof.md; the proof " + "carries verdicts, counts, relative input names, and sha256 " + "digests only (no transcript text, no audio bytes, no absolute " + "paths), plus a content_id computed over the canonical JSON " + "without the content_id field, so the same inputs under a " + "pinned SOURCE_DATE_EPOCH regenerate the same address." + ), + epilog=( + _exit_codes_epilog("prove") + "\n\n" + "Examples:\n" + " hotato prove --contracts contracts/ --gauntlet\n" + " hotato prove --contracts contracts/ --suite ci.suite.yaml " + "--agent support-v3\n" + " hotato prove --before before/ --after after/ --min-n 5 " + "--name support-v3-rc2 --format json\n" + " hotato prove --contracts contracts/ --out proofs/rc2/" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + pv.add_argument("--contracts", default=None, metavar="DIR", + help="contracts lane: re-score a directory of hotato " + "contracts (or one bundle) against each contract's " + "recorded policy, exactly as hotato contract verify " + "does") + pv.add_argument("--suite", default=None, metavar="SUITE.json", + help="suite lane: run a suite.v1 of conversation-tests " + "offline through the deterministic suite runner " + "(requires --agent)") + pv.add_argument("--agent", default=None, metavar="NAME", + help="agent id the suite lane runs against (required " + "with --suite)") + pv.add_argument("--before", default=None, metavar="RUN.json|DIR", + help="before/after lane: the OLD run envelope(s) " + "(requires --after)") + pv.add_argument("--after", default=None, metavar="RUN.json|DIR", + help="before/after lane: the NEW run envelope(s) " + "(requires --before)") + pv.add_argument("--min-n", type=int, default=3, + help="minimum previously-failing fixtures the " + "before/after lane needs to support its claim " + "(default 3); below it the lane is inconclusive, " + "not a pass") + pv.add_argument("--gauntlet", action="store_true", + help="gauntlet lane: run the bundled deterministic " + "turn-taking stress suite") + pv.add_argument("--name", default=None, metavar="NAME", + help="name the proof (letters, digits, dot, underscore, " + "hyphen); also the default output directory name " + "(default 'proof')") + pv.add_argument("--out", default=None, metavar="DIR", + help="directory for proof.json + proof.md (default " + ".hotato/proofs//)") + _add_format_arg(pv) + pv.set_defaults(func=_cmd_prove) + # --- loop: one-command orchestration of the closed loop, with memory ---- lp = sub.add_parser( "loop", diff --git a/src/hotato/intro.py b/src/hotato/intro.py index f4ccdae..e3036a1 100644 --- a/src/hotato/intro.py +++ b/src/hotato/intro.py @@ -76,7 +76,7 @@ def render(*, color: bool = False, width: int | None = None) -> str: out.append("") out.append(b + e + "hotato" + r + d + " (a hot potato)" + r) out.append("") - out.append(" " + b + "The local-first AI engineering platform." + r) + out.append(" " + b + "Local-first testing and observability for AI agents." + r) out.append(" Trace, evaluate, test, and gate any AI agent, on your machine.") out.append("") out.append(" " + b + "Try it now." + r + " One command, on your machine:") diff --git a/src/hotato/prove.py b/src/hotato/prove.py new file mode 100644 index 0000000..7d043bb --- /dev/null +++ b/src/hotato/prove.py @@ -0,0 +1,424 @@ +"""``hotato prove``: compose the evidence lanes you already ran into ONE +portable, content-addressed release proof. + +It adds no new scoring engine: every number here is one existing command +already measures -- ``hotato contract verify`` (the contracts lane, via +:func:`hotato.contract.verify_contracts`), ``hotato suite run`` (the suite +lane, via :func:`hotato.suite_run.run_suite`), ``hotato verify`` (the +before/after lane, via :func:`hotato.verify.verify_sides`), and ``hotato +gauntlet`` (the bundled stress suite, via +:func:`hotato.gauntlet.run_gauntlet`). :func:`run_prove` imports and calls the +same module functions those commands dispatch to; it never recomputes or +re-implements a verdict. + +The verdict is FAIL-CLOSED: + +* ``pass`` -- EVERY activated lane passed. This is the only exit-0 outcome. +* ``fail`` -- any lane failed or regressed. +* ``inconclusive`` -- no lane failed, but at least one refused its input or + came back inconclusive (a below ``--min-n`` battery, an unusable contracts + directory). Exits non-zero, so CI never reads "could not tell" as green. +* Zero activated lanes is a usage error (exit 2): a proof of nothing is + refused, not an empty pass. + +Share-safe by construction: the proof carries verdicts, counts, relative +input names, and sha256 digests only -- no transcript text, no audio bytes, +no absolute path, no environment value. ``content_id`` is a full content +address (the failure-record pattern): sha256 over the canonical proof JSON +without the ``content_id`` field, so the same inputs under a pinned +``SOURCE_DATE_EPOCH`` regenerate the same bytes and the same address. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from typing import Any, Dict, List, Optional + +from . import __version__ +from .errors import open_regular + +__all__ = [ + "SCHEMA_VERSION", + "LANES", + "run_prove", + "compute_content_id", + "serialize", + "render_text", + "render_md", +] + +SCHEMA_VERSION = "hotato.proof.v1" + +# The fixed lane order (activated lanes appear in this order in ``lanes``). +LANES = ("contracts", "suite", "verify", "gauntlet") + +VERDICT_PASS = "pass" +VERDICT_FAIL = "fail" +VERDICT_INCONCLUSIVE = "inconclusive" +VERDICT_REFUSED = "refused" + +EXIT_PASS = 0 +EXIT_FAIL = 1 +EXIT_INCONCLUSIVE = 2 + +_SAFE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,119}$") + + +# ========================================================================= +# canonical JSON + content address (the failure_record.py pattern) +# ========================================================================= + +def _canonical_json_bytes(obj: Any) -> bytes: + return json.dumps( + obj, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False, + ).encode("utf-8") + + +def _digest_bytes(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def compute_content_id(proof: Dict[str, Any]) -> str: + """The proof's content address: sha256 over the canonical JSON of every + field EXCEPT ``content_id`` itself (sorted keys, no insignificant + whitespace, UTF-8, finite numbers only).""" + identity = {k: v for k, v in proof.items() if k != "content_id"} + return _digest_bytes(_canonical_json_bytes(identity)) + + +def serialize(proof: Dict[str, Any]) -> str: + """The single proof.json serialization: deterministic (sorted keys, + 2-space indent, trailing newline) so the same proof dict always writes + the same bytes.""" + return json.dumps( + proof, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False, + ) + "\n" + + +# ========================================================================= +# share-safe input evidence: relative name + sha256 of the file / the +# directory's own manifest (relative paths + per-file digests) +# ========================================================================= + +def _file_sha256(path: str) -> str: + h = hashlib.sha256() + with open_regular(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return "sha256:" + h.hexdigest() + + +def _dir_manifest_digest(path: str) -> str: + """Digest of the directory's manifest: the sorted list of + ``{path: , sha256: }`` rows, + canonically serialized. Only the digest ships; the manifest (which + contains only relative paths) is not embedded.""" + rows = [] + for base, _dirs, files in os.walk(path): + for fname in files: + full = os.path.join(base, fname) + rel = os.path.relpath(full, path).replace(os.sep, "/") + rows.append({"path": rel, "sha256": _file_sha256(full)}) + rows.sort(key=lambda r: r["path"]) + return _digest_bytes(_canonical_json_bytes(rows)) + + +def _input_evidence(path: str) -> Dict[str, Any]: + """One evidence entry for a lane input: the input's RELATIVE name (its + basename -- never the absolute path) plus the sha256 digest of the file, + or of the directory's manifest. A missing input carries a null digest; + the lane's own refusal states why.""" + name = os.path.basename(os.path.normpath(path)) or "input" + digest: Optional[str] = None + try: + if os.path.isdir(path): + digest = _dir_manifest_digest(path) + elif os.path.isfile(path): + digest = _file_sha256(path) + except OSError: + digest = None + return {"input": name, "digest": digest} + + +# ========================================================================= +# the four lanes -- each one calls the EXISTING module function the CLI +# already dispatches to, and projects verdict + native counts only +# ========================================================================= + +def _lane(lane: str, verdict: str, counts: Dict[str, Any], + evidence: List[Dict[str, Any]], + refusal: Optional[str] = None) -> Dict[str, Any]: + entry = {"lane": lane, "verdict": verdict, "counts": counts, + "evidence": evidence} + if refusal is not None: + entry["refusal"] = refusal + return entry + + +def _lane_contracts(path: str) -> Dict[str, Any]: + from . import contract as _contract + + evidence = [_input_evidence(path)] + try: + res = _contract.verify_contracts(path) + except ValueError: + # verify_contracts raises for a missing/corrupt contract or a + # directory with no contracts. The message can carry the caller's + # path, so a FIXED share-safe reason ships instead. + return _lane("contracts", VERDICT_REFUSED, {}, evidence, + refusal="contract verify refused the input: no usable " + "contracts (missing, corrupt, or an empty " + "directory)") + counts = { + "contracts": res["count"], + "passed": res["summary"]["passed"], + "failed": res["summary"]["failed"], + "tampered": res["tampered"], + "refused": res["refused"], + "assertions_failed": res["assertions_failed"], + } + verdict = VERDICT_PASS if res["exit_code"] == 0 else VERDICT_FAIL + return _lane("contracts", verdict, counts, evidence) + + +def _lane_suite(suite_path: str, agent: str) -> Dict[str, Any]: + from . import suite_run as _suite_run + + evidence = [_input_evidence(suite_path)] + try: + suite_doc, base_dir = _suite_run.load_suite_file(suite_path) + res = _suite_run.run_suite( + suite_doc, base_dir, agent_id=agent, registry=None, + ) + except ValueError: + return _lane("suite", VERDICT_REFUSED, {}, evidence, + refusal="the suite runner refused the input: a " + "malformed suite.v1 file or an unresolvable " + "test ref") + counts = { + "tests": res["counts"]["tests"], + "runs": res["counts"]["runs"], + "passed_tests": res["counts"]["passed_tests"], + "failed_tests": res["counts"]["failed_tests"], + "refused_tests": res["counts"]["refused_tests"], + "simulator_invalid": res["counts"]["simulator_invalid"], + } + if res["exit_code"] == 0: + verdict = VERDICT_PASS + elif res["exit_code"] == 1: + verdict = VERDICT_FAIL + else: + verdict = VERDICT_REFUSED + return _lane("suite", verdict, counts, evidence) + + +def _lane_verify(before: str, after: str, min_n: int) -> Dict[str, Any]: + from . import verify as _verify + + evidence = [_input_evidence(before), _input_evidence(after)] + try: + res = _verify.verify_sides(before, after, min_n=min_n) + except ValueError: + return _lane("verify", VERDICT_REFUSED, {}, evidence, + refusal="verify refused the input: unusable run " + "envelopes or no fixtures pair between the " + "before and after sides") + axis = res["regression_axis"] + counts = { + "paired": res["paired"], + "used_to_fail": axis["used_to_fail"], + "fixed": axis["now_pass"], + "still_fail": axis["still_fail"], + "regressed": len(res["regressions"]), + "min_n": res["min_n"], + "claim_supported": bool(res["claim"]["supported"]), + } + # The same fail-closed reading hotato fix trial applies to verify's + # rollup: any regression fails; a below-min-n or zero-improvement + # battery is inconclusive, never a soft pass. + if res["regressions"]: + verdict = VERDICT_FAIL + elif not res["claim"]["supported"] or axis["now_pass"] == 0: + verdict = VERDICT_INCONCLUSIVE + else: + verdict = VERDICT_PASS + return _lane("verify", verdict, counts, evidence) + + +def _lane_gauntlet() -> Dict[str, Any]: + from . import gauntlet as _gauntlet + + manifest = _gauntlet.load_manifest() + res = _gauntlet.run_gauntlet(out_dir=None) + evidence = [{ + "input": res["suite"], + "digest": _digest_bytes(_canonical_json_bytes(manifest)), + }] + counts = {"passed": res["passed"], "total": res["total"]} + verdict = VERDICT_PASS if res["all_passed"] else VERDICT_FAIL + return _lane("gauntlet", verdict, counts, evidence) + + +# ========================================================================= +# composition: activated lanes -> one proof envelope +# ========================================================================= + +def _overall(lanes: List[Dict[str, Any]]) -> str: + verdicts = [entry["verdict"] for entry in lanes] + if any(v == VERDICT_FAIL for v in verdicts): + return VERDICT_FAIL + if all(v == VERDICT_PASS for v in verdicts): + return VERDICT_PASS + return VERDICT_INCONCLUSIVE + + +def _deterministic_created_at() -> str: + # The repo's reproducible-timestamp convention (simulate.py): + # $SOURCE_DATE_EPOCH when set, else a fixed instant -- never the wall + # clock, so the same inputs write byte-identical proof.json. + from . import simulate as _simulate + + return _simulate.deterministic_created_at(None) + + +def run_prove( + *, + contracts: Optional[str] = None, + suite: Optional[str] = None, + agent: Optional[str] = None, + before: Optional[str] = None, + after: Optional[str] = None, + min_n: int = 3, + gauntlet: bool = False, + name: Optional[str] = None, +) -> Dict[str, Any]: + """Run every activated evidence lane through its existing module function + and compose the results into one ``hotato.proof.v1`` dict (including + ``exit_code`` and ``content_id``). Raises ``ValueError`` (the CLI's + exit-2 usage path) when zero lanes are activated or a flag pair is + incomplete -- before anything runs and before anything is written.""" + if suite is not None and agent is None: + raise ValueError("--suite needs --agent NAME: the suite lane records " + "which agent the suite ran against") + if agent is not None and suite is None: + raise ValueError("--agent only accompanies --suite SUITE.json") + if (before is None) != (after is None): + raise ValueError("the before/after lane needs BOTH --before and " + "--after (each a run envelope JSON or a directory " + "of them)") + if contracts is None and suite is None and before is None and not gauntlet: + raise ValueError( + "no evidence lane activated. hotato prove composes lanes you " + "already have: --contracts DIR, --suite SUITE.json --agent NAME, " + "--before RUN --after RUN, and/or --gauntlet. A proof of nothing " + "is refused, not an empty pass." + ) + proof_name = name if name is not None else "proof" + if not _SAFE_NAME_RE.match(proof_name): + raise ValueError( + "--name must be letters, digits, dot, underscore, or hyphen " + "(leading alphanumeric, at most 120 chars); it names the proof " + "and its default output directory" + ) + + lanes: List[Dict[str, Any]] = [] + if contracts is not None: + lanes.append(_lane_contracts(contracts)) + if suite is not None: + lanes.append(_lane_suite(suite, agent)) + if before is not None: + lanes.append(_lane_verify(before, after, min_n)) + if gauntlet: + lanes.append(_lane_gauntlet()) + + overall = _overall(lanes) + exit_code = {VERDICT_PASS: EXIT_PASS, VERDICT_FAIL: EXIT_FAIL, + VERDICT_INCONCLUSIVE: EXIT_INCONCLUSIVE}[overall] + proof = { + "tool": "hotato", + "schema_version": SCHEMA_VERSION, + "name": proof_name, + "hotato_version": __version__, + "created_at": _deterministic_created_at(), + "lanes": lanes, + "overall": overall, + "exit_code": exit_code, + } + proof["content_id"] = compute_content_id(proof) + return proof + + +# ========================================================================= +# rendering (deterministic given the proof dict) +# ========================================================================= + +def _counts_cell(counts: Dict[str, Any]) -> str: + if not counts: + return "-" + return " ".join(f"{k}={v}" for k, v in counts.items()) + + +def _table_rows(proof: Dict[str, Any]) -> List[Dict[str, str]]: + rows = [] + for entry in proof["lanes"]: + rows.append({ + "lane": entry["lane"], + "verdict": entry["verdict"], + "counts": _counts_cell(entry["counts"]), + }) + return rows + + +def render_text(proof: Dict[str, Any], + proof_path: Optional[str] = None) -> str: + """The tight per-lane table + overall verdict + content address (+ where + the proof landed, when the caller wrote it).""" + lines = [ + f"hotato prove: {proof['name']} -- overall " + f"{proof['overall'].upper()} (exit {proof['exit_code']})", + ] + rows = _table_rows(proof) + lane_w = max(len("lane"), *(len(r["lane"]) for r in rows)) + verdict_w = max(len("verdict"), *(len(r["verdict"]) for r in rows)) + lines.append(f" {'lane':<{lane_w}} {'verdict':<{verdict_w}} counts") + for r in rows: + lines.append( + f" {r['lane']:<{lane_w}} {r['verdict']:<{verdict_w}} " + f"{r['counts']}" + ) + lines.append(f"content_id: {proof['content_id']}") + if proof_path: + lines.append(f"proof: {proof_path}") + return "\n".join(lines) + "\n" + + +def render_md(proof: Dict[str, Any]) -> str: + """proof.md: the same per-lane table, overall verdict, and content + address as the text rendering, in markdown.""" + lines = [ + f"# hotato proof: {proof['name']}", + "", + f"Overall: **{proof['overall'].upper()}** " + f"(exit {proof['exit_code']}). Pass requires every activated lane " + "to pass; an inconclusive or refused lane exits non-zero.", + "", + "| lane | verdict | counts |", + "| --- | --- | --- |", + ] + for r in _table_rows(proof): + lines.append(f"| {r['lane']} | {r['verdict']} | {r['counts']} |") + lines += [ + "", + f"- hotato_version: {proof['hotato_version']}", + f"- created_at: {proof['created_at']}", + f"- content_id: `{proof['content_id']}`", + "", + "Every number above is one existing command's own measurement " + "(contract verify, suite run, verify, gauntlet); the proof carries " + "verdicts, counts, relative input names, and sha256 digests only.", + ] + return "\n".join(lines) + "\n" diff --git a/tests/test_capture.py b/tests/test_capture.py index 2cb0595..749dcf6 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -175,7 +175,7 @@ def test_bare_hotato_shows_the_first_run_screen(capsys): assert rc == 0 out = capsys.readouterr().out low = out.lower() - assert "the local-first ai engineering platform" in low + assert "local-first testing and observability for ai agents" in low assert "hotato start --demo" in out assert "hotato serve" in out assert "hotato.dev" in low diff --git a/tests/test_describe_cli.py b/tests/test_describe_cli.py index 6f641a2..6b1865a 100644 --- a/tests/test_describe_cli.py +++ b/tests/test_describe_cli.py @@ -50,7 +50,7 @@ "simulate", "compare", "scan", "synth", "battery", "battery robustness", "gauntlet", "gauntlet badge", "trust", - "ingest", "analyze", "verify", "fix", "fix trial", "loop", + "ingest", "analyze", "verify", "fix", "fix trial", "prove", "loop", "investigate", "investigate label", "describe", "init", "init webhook", "init starter", "init ci", "issue", "issue create", diff --git a/tests/test_positioning_lockstep.py b/tests/test_positioning_lockstep.py index 8083d0e..0249490 100644 --- a/tests/test_positioning_lockstep.py +++ b/tests/test_positioning_lockstep.py @@ -4,12 +4,13 @@ across surfaces. test_version_lockstep guards version NUMBERS; this guards the POSITIONING COPY so they can never drift apart again. -The invariant (2026-07-21 repositioning, WAVE 1): every user-facing surface -calls hotato "the local-first AI engineering platform" (trace, evaluate, test, +The invariant (2026-07-22 repositioning): every user-facing surface calls hotato +"local-first testing and observability for AI agents" (trace, evaluate, test, and gate any AI agent), and NONE of them still leads with a retired narrower product definition (turn-taking eval, flight recorder, conversation QA for -voice agents, or regression testing for voice agents). The turn-taking wedge -survives as a capability/example, not as the product's identity. +voice agents, regression testing for voice agents, or the vague "AI engineering +platform"). The turn-taking wedge survives as a capability/example, not as the +product's identity. """ import pathlib @@ -28,7 +29,7 @@ ] # The canonical positioning phrase every surface must carry. -_POSITIONING = "ai engineering platform" +_POSITIONING = "testing and observability for ai agents" # Retired product DEFINITIONS that must not survive as the lead identity. These # are exact product-definition phrases, not the individual words -- "voice", @@ -39,6 +40,7 @@ "Offline turn-taking analysis and regression evidence for dual-channel", "conversation QA for voice agents", "regression testing for voice agents", + "AI engineering platform", ] @@ -53,8 +55,8 @@ def test_every_surface_says_the_platform(): if _POSITIONING not in full: missing.append(rel) assert not missing, ( - "these surfaces no longer call hotato the local-first AI engineering " - f"platform (positioning drift): {missing}") + "these surfaces no longer carry the positioning 'local-first testing " + f"and observability for AI agents' (positioning drift): {missing}") def test_no_surface_leads_with_a_retired_product_definition(): @@ -66,7 +68,7 @@ def test_no_surface_leads_with_a_retired_product_definition(): hits.append(f"{rel}: {phrase!r}") assert not hits, ( "a surface still leads with a retired narrow product definition; " - f"reposition it to the AI engineering platform: {hits}") + f"reposition it to local-first testing and observability for AI agents: {hits}") def test_the_five_dimensions_are_named_consistently(): diff --git a/tests/test_prove.py b/tests/test_prove.py new file mode 100644 index 0000000..6922480 --- /dev/null +++ b/tests/test_prove.py @@ -0,0 +1,270 @@ +"""``hotato prove``: the top-level release proof that composes the existing +evidence lanes (contracts / suite / before-after / gauntlet) into one +fail-closed, content-addressed proof.json + proof.md. + +Pinned here: + + * a green contracts lane -> overall pass, exit 0, proof.json + proof.md + written with every schema field present; + * a failing contract (the ``start --demo`` bundle) -> overall fail, exit 1; + * zero activated lanes -> exit 2 usage error and NOTHING written (a proof + of nothing is refused, not an empty pass); + * a refused-only run (an empty contracts dir) -> overall inconclusive with + the lane refused, exit 2, never exit 0; + * determinism: two runs over the same inputs under a pinned + SOURCE_DATE_EPOCH write byte-identical proof.json (same content_id); + * share-safety: no absolute path appears anywhere in the serialized proof; + * composition honesty: the contracts lane's counts equal what + hotato.contract.verify_contracts itself reports on the same directory -- + prove adds no scoring engine of its own. +""" + +from __future__ import annotations + +import json +from importlib import resources + +import pytest + +from hotato import cli +from hotato import contract as _contract +from hotato import prove as _prove + +HARD = str(resources.files("hotato").joinpath( + "data", "audio", "01-hard-interruption.example.wav")) # yields at 2.40 + + +@pytest.fixture(autouse=True) +def _isolated_home(tmp_path, monkeypatch): + # The demo-contract fixture path (`start --demo`) reads HOTATO_HOME; keep + # every run hermetic, exactly as tests/test_start_cli.py does. + monkeypatch.setenv("HOTATO_HOME", str(tmp_path / "home")) + + +def _passing_contracts_dir(tmp_path): + """One passing contract, created with the existing `contract create` + machinery (the tests/test_contract_cli.py pattern): the bundled + hard-interruption example yields at 2.40, so expect=yield passes.""" + cdir = tmp_path / "contracts" + cdir.mkdir() + rc = cli.main([ + "contract", "create", "--stereo", HARD, "--id", "prove-pass-001", + "--onset", "2.40", "--expect", "yield", "--out", str(cdir), + ]) + assert rc == 0 + return cdir + + +def _failing_contracts_dir(tmp_path): + """The demo failure contract `start --demo` creates (FAIL as expected): + the existing exit-1 gate for `hotato contract verify contracts/`.""" + demo_dir = tmp_path / "demo" + demo_dir.mkdir() + rc = cli.main(["start", "--demo", "--dir", str(demo_dir)]) + assert rc == 0 + cdir = demo_dir / "contracts" + assert (cdir / "demo-missed-interruption.hotato" / "contract.json").is_file() + return cdir + + +def _read_proof(out_dir): + with open(out_dir / "proof.json", encoding="utf-8") as fh: + return json.load(fh) + + +# --- 1. green contracts lane -> pass, exit 0, both files, schema fields ---- + +def test_green_contracts_lane_passes_and_writes_both_files(tmp_path, capsys): + cdir = _passing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + rc = cli.main(["prove", "--contracts", str(cdir), "--out", str(out)]) + assert rc == 0 + assert (out / "proof.json").is_file() + assert (out / "proof.md").is_file() + + proof = _read_proof(out) + assert proof["tool"] == "hotato" + assert proof["schema_version"] == "hotato.proof.v1" + assert proof["name"] == "proof" + assert proof["hotato_version"] + assert proof["created_at"] + assert proof["overall"] == "pass" + assert proof["exit_code"] == 0 + assert proof["content_id"].startswith("sha256:") + assert len(proof["lanes"]) == 1 + lane = proof["lanes"][0] + assert lane["lane"] == "contracts" + assert lane["verdict"] == "pass" + assert lane["counts"]["passed"] == 1 + assert lane["counts"]["failed"] == 0 + assert lane["evidence"][0]["input"] == "contracts" + assert lane["evidence"][0]["digest"].startswith("sha256:") + + # The text surface carries the lane table, the overall verdict, the + # content address, and where the proof landed; proof.md mirrors it. + text = capsys.readouterr().out + assert "overall PASS" in text + assert proof["content_id"] in text + assert "proof.json" in text + md = (out / "proof.md").read_text(encoding="utf-8") + assert "PASS" in md + assert proof["content_id"] in md + assert "| contracts | pass |" in md + + +def test_content_id_matches_the_record_pattern(tmp_path): + cdir = _passing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + assert cli.main(["prove", "--contracts", str(cdir), "--out", str(out)]) == 0 + proof = _read_proof(out) + # sha256 over the canonical JSON WITHOUT the content_id field (the + # failure-record content-address pattern). + assert _prove.compute_content_id(proof) == proof["content_id"] + + +# --- 2. failing contract -> overall fail, exit 1 --------------------------- + +def test_failing_contract_fails_the_proof(tmp_path): + cdir = _failing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + rc = cli.main(["prove", "--contracts", str(cdir), "--out", str(out)]) + assert rc == 1 + proof = _read_proof(out) + assert proof["overall"] == "fail" + assert proof["exit_code"] == 1 + lane = proof["lanes"][0] + assert lane["verdict"] == "fail" + assert lane["counts"]["failed"] >= 1 + + +# --- 3. zero lanes -> exit 2, nothing written ------------------------------ + +def test_zero_lanes_is_a_usage_error_and_writes_nothing(tmp_path, capsys): + out = tmp_path / "proofout" + rc = cli.main(["prove", "--out", str(out)]) + assert rc == 2 + assert not out.exists() + err = capsys.readouterr().err + assert "no evidence lane activated" in err + + +def test_suite_without_agent_is_a_usage_error(tmp_path): + out = tmp_path / "proofout" + rc = cli.main(["prove", "--suite", str(tmp_path / "s.json"), + "--out", str(out)]) + assert rc == 2 + assert not out.exists() + + +def test_before_without_after_is_a_usage_error(tmp_path): + out = tmp_path / "proofout" + rc = cli.main(["prove", "--before", str(tmp_path / "b.json"), + "--out", str(out)]) + assert rc == 2 + assert not out.exists() + + +# --- 4. refused-only -> inconclusive, exit 2, never 0 ---------------------- + +def test_refused_only_is_inconclusive_never_a_pass(tmp_path): + empty = tmp_path / "contracts" + empty.mkdir() + out = tmp_path / "proofout" + rc = cli.main(["prove", "--contracts", str(empty), "--out", str(out)]) + assert rc == 2 + assert rc != 0 + proof = _read_proof(out) + assert proof["overall"] == "inconclusive" + assert proof["exit_code"] == 2 + lane = proof["lanes"][0] + assert lane["verdict"] == "refused" + assert lane["counts"] == {} + assert "refusal" in lane + + +# --- 5. determinism under a pinned SOURCE_DATE_EPOCH ----------------------- + +def test_pinned_epoch_makes_proof_json_byte_identical(tmp_path, monkeypatch): + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + cdir = _passing_contracts_dir(tmp_path) + out_a = tmp_path / "proof-a" + out_b = tmp_path / "proof-b" + assert cli.main(["prove", "--contracts", str(cdir), + "--out", str(out_a)]) == 0 + assert cli.main(["prove", "--contracts", str(cdir), + "--out", str(out_b)]) == 0 + a = (out_a / "proof.json").read_bytes() + b = (out_b / "proof.json").read_bytes() + assert a == b + assert json.loads(a)["created_at"] == "2023-11-14T22:13:20Z" + + +# --- 6. share-safety: no absolute path in the serialized proof ------------- + +def test_proof_json_carries_no_absolute_path(tmp_path): + cdir = _passing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + assert cli.main(["prove", "--contracts", str(cdir), "--out", str(out)]) == 0 + raw = (out / "proof.json").read_text(encoding="utf-8") + assert str(tmp_path) not in raw + assert str(cdir) not in raw + + +def test_refused_proof_carries_no_absolute_path_either(tmp_path): + # The refusal path is the one that could leak the caller's path through + # the underlying ValueError message; the lane ships a fixed reason. + empty = tmp_path / "contracts" + empty.mkdir() + out = tmp_path / "proofout" + assert cli.main(["prove", "--contracts", str(empty), + "--out", str(out)]) == 2 + raw = (out / "proof.json").read_text(encoding="utf-8") + assert str(tmp_path) not in raw + + +# --- 7. composition honesty: the lane equals verify_contracts itself ------- + +def test_contracts_lane_equals_verify_contracts_on_the_same_dir(tmp_path): + cdir = _passing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + assert cli.main(["prove", "--contracts", str(cdir), "--out", str(out)]) == 0 + lane = _read_proof(out)["lanes"][0] + + res = _contract.verify_contracts(str(cdir)) + assert lane["counts"]["contracts"] == res["count"] + assert lane["counts"]["passed"] == res["summary"]["passed"] + assert lane["counts"]["failed"] == res["summary"]["failed"] + assert lane["counts"]["tampered"] == res["tampered"] + assert lane["counts"]["refused"] == res["refused"] + assert lane["counts"]["assertions_failed"] == res["assertions_failed"] + assert (lane["verdict"] == "pass") == (res["exit_code"] == 0) + + +# --- json format parity ---------------------------------------------------- + +def test_format_json_prints_the_proof_envelope(tmp_path, capsys): + cdir = _passing_contracts_dir(tmp_path) + out = tmp_path / "proofout" + capsys.readouterr() # drop the `contract create` text output + rc = cli.main(["prove", "--contracts", str(cdir), "--out", str(out), + "--format", "json"]) + assert rc == 0 + printed = json.loads(capsys.readouterr().out) + assert printed == _read_proof(out) + + +def test_gauntlet_lane_passes_and_counts_the_bundled_suite(tmp_path, capsys): + # The gauntlet lane runs the bundled deterministic stress suite through + # gauntlet.run_gauntlet itself; on the shipped scorer it passes 10/10, so + # the proof is a pass with the suite's own counts as evidence. + out = tmp_path / "proofout" + rc = cli.main(["prove", "--gauntlet", "--out", str(out)]) + assert rc == 0 + capsys.readouterr() + proof = _read_proof(out) + assert proof["overall"] == "pass" + (lane,) = proof["lanes"] + assert lane["lane"] == "gauntlet" + assert lane["verdict"] == "pass" + assert lane["counts"]["total"] == 10 + assert lane["counts"]["passed"] == 10