From 5162c0b798f4db4fadc022906642bf5b851032cf Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 09:18:19 -0500 Subject: [PATCH 1/3] feat(ai): provider prompt caching for agent() and llm(); cache usage per turn and run; a repeated-turn guard and a run deadline agent() asks the provider to cache each turn's prefix (default 'auto'; llm() 'off'): GenerateRequest takes system { stable, run } and promptCache { session? }, and PROMPT_CACHE_RULES decides what each wire and host is sent - cache_control blocks on the Anthropic wire, content-part marks for anthropic/* on OpenRouter, nothing for other models there or on OpenAI, x-session-id / prompt_cache_key for a session key. The chat-completions reader takes cache_write_tokens out of prompt_tokens; onTurn gains the run's running usage and cost. maxRepeatedToolCalls (default 3) stops a loop repeating the same calls and results (X_AGENT_TOOL_LOOP); deadlineMs bounds a run's wall clock (X_AGENT_DEADLINE); a caller abort mid-call is X_ABORTED. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 17 +- docs/history/ai.md | 38 +++ framework.manifest.json | 12 +- packages/ai/CLAUDE.md | 104 +++--- packages/ai/README.md | 64 ++++ packages/ai/src/agent-cache.test.ts | 302 +++++++++++++++++ packages/ai/src/agent-def.ts | 111 +++++++ packages/ai/src/agent-errors.ts | 42 +++ packages/ai/src/agent-guards.test.ts | 221 +++++++++++++ packages/ai/src/agent-guards.ts | 90 ++++++ packages/ai/src/agent-job.test.ts | 2 + packages/ai/src/agent-system.ts | 37 +++ packages/ai/src/agent-usage.ts | 27 ++ packages/ai/src/agent.test.ts | 6 + packages/ai/src/agent.ts | 151 ++++----- packages/ai/src/errors.ts | 4 + packages/ai/src/index.ts | 14 +- packages/ai/src/llm-prompt-cache.test.ts | 57 ++++ packages/ai/src/llm.ts | 11 + packages/ai/src/openai-body.ts | 39 ++- packages/ai/src/openai-messages.ts | 55 +++- packages/ai/src/openai-provider.ts | 13 +- packages/ai/src/openai-wire.test.ts | 39 +++ packages/ai/src/openai-wire.ts | 23 +- packages/ai/src/prompt-cache-blocks.ts | 61 ++++ packages/ai/src/prompt-cache.test.ts | 391 +++++++++++++++++++++++ packages/ai/src/prompt-cache.ts | 173 ++++++++++ packages/ai/src/provider.ts | 30 +- packages/http/src/error-map-tier-4.ts | 4 + wiki/Agents.md | 108 ++++++- wiki/Error-Codes.md | 2 + wiki/MCP-And-AI.md | 1 + 32 files changed, 2083 insertions(+), 166 deletions(-) create mode 100644 packages/ai/src/agent-cache.test.ts create mode 100644 packages/ai/src/agent-def.ts create mode 100644 packages/ai/src/agent-errors.ts create mode 100644 packages/ai/src/agent-guards.test.ts create mode 100644 packages/ai/src/agent-guards.ts create mode 100644 packages/ai/src/agent-system.ts create mode 100644 packages/ai/src/agent-usage.ts create mode 100644 packages/ai/src/llm-prompt-cache.test.ts create mode 100644 packages/ai/src/prompt-cache-blocks.ts create mode 100644 packages/ai/src/prompt-cache.test.ts create mode 100644 packages/ai/src/prompt-cache.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d726ad97b..37961c9a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,22 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ## [Unreleased] -Nothing yet. +### Added + +Tier 2 — http. Tier 4 — ai. + +- **Provider prompt caching: an `agent()` no longer pays input price for its stable prefix on every turn.** Every turn re-sends the whole transcript, and nothing asked a provider to cache it. A `GenerateRequest` now takes `system: { stable, run }` (two blocks; the stable one first and byte-identical across runs) and `promptCache: { session? }`, and each provider marks the stable system part and the transcript's tail the way its wire and host want — `PROMPT_CACHE_RULES` in `@ultimat3/ai`, a table, sources quoted in `prompt-cache.ts`: `cache_control: { type: 'ephemeral' }` on Anthropic content blocks (three of the four breakpoints), the same mark on content parts for an `anthropic/…` model on OpenRouter (two), and **no** mark for any other model there or on OpenAI, where caching is implicit and an explicit mark on Gemini is a paid write. A session key travels as OpenRouter's `x-session-id` header or OpenAI's `prompt_cache_key`; an endpoint the table does not name is sent nothing. `agent({ system, promptCache })` and `llm({ promptCache })` are the declaration's side: `'auto' | 'off' | { session }`, and `system()` returns the text loaded at run time as `{ stable?, run? }`. +- **Default: `agent()` caches (`promptCache: 'auto'`), `llm()` does not (`'off'`).** This is the one behaviour that changes for a caller who passes nothing: an agent's requests to an Anthropic model now carry `cache_control` marks (so `system` is sent as blocks or parts, not a string), a first turn is billed as a cache write (1.25x input on Anthropic) and later turns as reads, and the pre-flight cost estimate of a marked request prices the prompt as a write — a `budget.costPerCall` set within 25% of an agent's prompt cost now refuses. `promptCache: 'off'` restores the old request byte for byte. A one-shot `llm()` is unchanged. +- **Cache usage is reported per turn and per run.** The chat-completions reader takes `prompt_tokens_details.cache_write_tokens` as well as `cached_tokens` out of `prompt_tokens` (which includes both) and reports them as `cacheWriteTokens` / `cacheReadTokens`; it reported a write as full-price input before. `onTurn`'s event gains `run` — the running `usage` and `cost`, priced once from the summed usage — and the `agent.turn` span event gains `llm.cache.read_tokens` / `llm.cache.write_tokens`. Prices are the model row's existing `cacheReadPerMillion` / `cacheWritePerMillion`; a token budget counts read and written tokens one for one, as it always has. +- **`maxRepeatedToolCalls` — an agent that repeats itself is stopped (`X_AGENT_TOOL_LOOP`, 500).** That many consecutive turns making the same tool calls **and** reading the same results end the run. **Default 3, on for every agent**: a loop that used to end in `X_AGENT_MAX_TURNS` or `X_AI_BUDGET_EXCEEDED` after repeating one call now ends earlier with this code. A polled tool whose answer changes is not a repeat. +- **`deadlineMs` — a wall-clock ceiling for one agent run (`X_AGENT_DEADLINE`, 504).** No default. Passing it aborts the model call in flight on its socket, stops waiting for a tool still running, and starts no further turn. +- `@ultimat3/ai` exports `PROMPT_CACHE_RULES`, `promptCacheRule`, `systemText`, `SESSION_KEY_MAX`, `AgentToolLoopError`, `AgentDeadlineError` and the types `SystemPrompt`, `SystemParts`, `PromptCacheHint`, `PromptCacheOption`, `PromptCacheRule`, `CacheWire`, `AgentSystem`, `AgentRunUsage`. + +### Fixed + +Tier 4 — ai. + +- **A caller that goes away during an agent's model call is `X_ABORTED`.** The aborted socket surfaced as `X_AI_PROVIDER_UNAVAILABLE`, the one code the package classifies retryable, so a job retried a run its caller had abandoned. ## 27.8.1 - 2026-10-11 diff --git a/docs/history/ai.md b/docs/history/ai.md index 9cf5b93b9..79fea67e3 100644 --- a/docs/history/ai.md +++ b/docs/history/ai.md @@ -695,3 +695,41 @@ until 2026-08, naming a tool no catalog contained (`llm.test.ts`, `agent.test.ts - Metadata is bound `::text::jsonb`. A bound string cast straight to `::jsonb` is JSON-encoded twice, reads back correctly, and makes every `metadata ->> key` filter match nothing. + +## Provider prompt caching, and the unattended-run guards (2026-10) + +- **Why the loop marks the cache and an app does not.** Every turn of an `agent()` re-sends the + whole transcript, so a 10-turn run with an 8–20k-token stable prefix paid input price for that + prefix ten times. `openai-wire.ts` already READ `cached_tokens`; nothing sent a breakpoint. The + mark has to be placed by whoever builds the request, and that is the loop. +- **Why a rule table (`PROMPT_CACHE_RULES`) and not a flag on the provider.** The same + chat-completions wire needs three different things: an explicit mark for an Anthropic model on + OpenRouter, NO mark for Gemini there (an explicit one turns the free implicit cache into a paid + write), and nothing an unknown self-hosted server might reject. Host and model prefix decide, + first match wins, unknown gets nothing. +- **Why `agent()` defaults on and `llm()` off.** A write costs 1.25x input on Anthropic and a read + 0.1x or less: two turns over a prefix P cost 1.35P marked against 2P unmarked, and a tool loop is + two turns or more by construction. A one-shot `llm()` has no second request to read the cache. +- **Why the stable part is the FIRST system block and the only system block marked.** It is the + half other runs share; the per-run half after it is covered by the transcript's own tail mark. +- **Why `prompt_tokens` has both counters subtracted.** Measured against OpenRouter on 2026-10-11: + turn one reported `prompt_tokens: 19918` with `cache_write_tokens: 19914`, turn two + `prompt_tokens: 19988` with `cached_tokens: 19914` and `cache_write_tokens: 72`. +- **Why the pre-flight estimate prices a hinted prompt as a write.** It is the dearest the prompt + can be billed (turn one), and a ceiling checked against a cheaper guess is one a real call walks + through. +- **Why the repeated-turn guard compares the RESULT too.** Polling one tool with one argument is + legitimate while its answer changes. The same call with the same answer cannot be followed by a + different turn. Three ceiling tests that script a model repeating itself for ever lift the guard + (`maxRepeatedToolCalls: 100`) so the ceiling they are about is the one that stops the run. +- **Why `deadlineMs` has no default.** `GenerateRequest.signal`'s own rule: a timeout nobody + declared is a lost answer nothing reports. +- **Why a caller abort mid-call is rethrown as `X_ABORTED`.** The aborted socket surfaced as + `X_AI_PROVIDER_UNAVAILABLE`, the one code this package classifies retryable, so a job retried a + run its caller had abandoned. +- Condensed out of `packages/ai/CLAUDE.md` in the same change, to stay under its ceiling: the + default `BudgetStore` grew by one entry per caller when every call wrote its keys (hence + `takeScope` / `meteredKeys`); budget keys are absent for an `undefined`, `null` or `''` org, the + `orgless` case `query`'s cache uses; `RetryPolicy` / `DEFAULT_GATEWAY_RETRY` keep the field names + an app writes; the status tables and the OpenAI `estimatedUsage` fallback are the differences + `provider-parity.test.ts` asserts between the two wire formats. diff --git a/framework.manifest.json b/framework.manifest.json index b5fe89306..63ada0264 100644 --- a/framework.manifest.json +++ b/framework.manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "buildId": "e7c9d5de792d076bf2174805dbeec082841bf128e7692d3778cc31119ef57910", + "buildId": "9d86758bac163ca7288d402e246c6bb2f237212b09b82fe2b7601c9864e7e585", "tiers": { "0": [ "core", @@ -374,11 +374,21 @@ "owner": "cli", "at": "packages/cli/src/error-codes.ts" }, + { + "code": "X_AGENT_DEADLINE", + "owner": "ai", + "at": "packages/ai/src/errors.ts" + }, { "code": "X_AGENT_MAX_TURNS", "owner": "ai", "at": "packages/ai/src/errors.ts" }, + { + "code": "X_AGENT_TOOL_LOOP", + "owner": "ai", + "at": "packages/ai/src/errors.ts" + }, { "code": "X_AGENT_TOOL_UNEXPOSED", "owner": "ai", diff --git a/packages/ai/CLAUDE.md b/packages/ai/CLAUDE.md index d25f01c7f..909090319 100644 --- a/packages/ai/CLAUDE.md +++ b/packages/ai/CLAUDE.md @@ -28,7 +28,7 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe |---|---| | `models.ts` | the model REGISTRY: `registerModel`, limits, prices, the reasoning controls each one accepts — and NO row of its own | | `model-resolve.ts` | `resolveModel` / `modelForCall` — declaration → prompt → gateway `defaultModel`, none is `X_AI_MODEL_UNRESOLVED` naming all three; `describedModel` for a published fact | -| `model-fixture.ts` | the rows this package's own suites register (`useFixtureModels()`), as an app's `models.ts` would. Not shipped (`!src/**/*-fixture.ts`) | +| `model-fixture.ts` | the rows this package's own suites register (`useFixtureModels()`). Not shipped | | `provider.ts` | `Provider` interface, the request half, the money arithmetic, `anthropicProvider()` | | `echo-provider.ts` | `echoProvider()`, the deterministic double — split from `provider.ts` at its line ceiling | | `content-blocks.ts` | the `image` / `document` block shapes, the screen every request runs them through, their token estimate | @@ -62,6 +62,10 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe | `respond.ts` | the `respond` tool for one `output` (a non-object one wrapped in `{ value }`), its reader, and `parseJsonish` | | `llm-stream.ts` | `.stream()`'s plumbing: the sink, the ambient mark, the one-turn drive | | `agent.ts` | `agent()` — the tool loop, declared as an `action` | +| `agent-def.ts` / `agent-system.ts` / `agent-usage.ts` | the declaration's types; the system one run sends; its running usage | +| `agent-guards.ts` / `agent-errors.ts` | the repeated-turn detector and the run deadline; their two classes | +| `prompt-cache.ts` | PROVIDER prompt caching: `SystemParts`, `PromptCacheHint`, `PROMPT_CACHE_RULES` (sources in its header) | +| `prompt-cache-blocks.ts` | the Anthropic wire's marks (chat-completions': `openai-messages.ts`) | | `agent-transcript.ts` | what one turn leaves in the transcript: the assistant replay, the tool results, the correction | | `agent-facts.ts` | `describeAgents()` — the agent registry, and the row a manifest WOULD publish: nothing reads it yet, because `manifest` is tier 4 like this package and the consumer has to be `cli` at tier 5 | | `agent-job.ts` | `agentJob()` — an agent as a real `JobHandle`, composed from `job()` | @@ -71,37 +75,30 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe | `redaction.ts` | the one gate between `vars()` and the provider: a `Secret` never reaches a prompt | | `eval-errors.ts` | the five `X_EVAL_*` classes; their codes and titles stay in `errors.ts` | | `llm-cache.ts` | the semantic cache half of `llm()`: what a declaration may partition on, and the store it reaches | -| `llm-fixture.ts` | the harness `llm.test.ts` and `llm-cache.test.ts` share. Not shipped (`!src/**/*-fixture.ts`) | -| `bounds-fixture.ts` | the one `refusal`/`asyncRefusal` every numeric-bound suite here asserts through. Not shipped, same rule | -| `agent-bounds.test.ts` | the agent loop's ceilings (split from `agent.test.ts`, at its line ceiling) | +| `llm-fixture.ts` / `bounds-fixture.ts` | the `llm()` harness; the one `refusal`/`asyncRefusal` of every bound suite. Not shipped (`!src/**/*-fixture.ts`) | | `runtime.ts` | the ambient gateway / embedder / semantic caches an `llm()` reaches | -| `fix-line-fixture.ts` / `fix-line.evals-fixture.ts` / `fix-line.v1.baseline.json` / `fix-line.eval.test.ts` | the package's dogfood eval — proof the baseline convention fails a build | +| `fix-line*` | the package's dogfood eval — proof the baseline convention fails a build | ## Invariants — bounds and budget -- **Every numeric option is screened; `??` is not the screen.** `finiteOption`/`finiteCount` from - `@ultimat3/core` are the one form; `bun run finite-bounds` is the ratchet, and it cannot see a - required option with no `??`, a default parameter or a cross-field product. **`maxTokens` is the - worst**: it IS the budget's pre-flight estimate, and a `NaN` switches every actor and org ceiling - off. `Gateway.generate`/`stream` screen it at the one seam, `registerModel` screens `maxOutput`, and - `llm()`/`agent()` screen theirs at DECLARATION. +- **Every numeric option is screened; `??` is not the screen** (`finiteOption`/`finiteCount`; + `bun run finite-bounds` is the ratchet, blind to a required option, a default parameter and a + cross-field product). **`maxTokens` is the worst**: it IS the budget's pre-flight estimate, and a + `NaN` switches every ceiling off — the gateway, `registerModel` (`maxOutput`) and `llm()`/`agent()` + (at DECLARATION) each screen theirs. - **A bound is screened under the key the DECLARATION uses** (`budget.tokensPerRun` in `limitsOf` before the ledger's `request`; `retrieve()`'s own `k`; `remoteEmbedder()`'s `maxResponseBytes`). - A per-call budget `derive`s from the ambient ledger and only TIGHTENS. **A derived ledger reports back up the chain** (every debit on every ancestor; `reserve` checks each `request` scope); the STORE is written once, by the ledger the call was made on. -- **`reserve` debits, then checks — never a read-then-write across an `await`.** The in-memory scopes - check and debit synchronously; the store's go through `BudgetStore.take(key, tokens, limit)`, which - is REQUIRED and must be atomic (no default over `spent` + `add` — that pair is the race). A refusal - at a later scope gives back what the earlier ones took. -- **The store is touched only for a scope whose ceiling is DECLARED** (`takeScope`, `meteredKeys`): - every call carries keys, so a write per key grew the default store by one entry per caller. - `memoryBudgetStore()` has no window and no eviction (eviction is a ceiling bypass); a counter back - at zero is deleted. +- **`reserve` debits, then checks — never a read-then-write across an `await`.** The store's scopes + go through `BudgetStore.take(key, tokens, limit)`: REQUIRED, atomic (never `spent` + `add`). A + refusal at a later scope gives back what the earlier ones took. +- **The store is touched only for a scope whose ceiling is DECLARED** (`takeScope`, `meteredKeys`). + `memoryBudgetStore()` has no window and no eviction (a ceiling bypass); a zero counter is deleted. - **`llm()`, `agent()`, `hive()` root as `currentBudget() ?? gateway.callLedger(budgetKeysFor(ctx.actor))`** - — the gateway's ceilings, keyed `actor::` / `org:` (none for an `undefined`/`null`/`''` - org — the `orgless` test `query`'s cache uses). `callLedger` is required on - `Gateway`; `hive()` asks `installedGateway()` so a hive of plain actions needs none. + — the gateway's ceilings, keyed `actor::` / `org:` (none for an empty org). + `callLedger` is required on `Gateway`; `hive()` asks `installedGateway()`. - **`BudgetStore` is where `actor` and `org` live; the default is per PROCESS** (`providerGateway({ budgetStore })`). `add` takes a negative `tokens` for a release — a store that clamps at zero leaks the ceiling. @@ -131,10 +128,9 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe - The gateway is ambient (`configureAi`); absent at call time is `X_AI_GATEWAY_MISSING`, never a default provider. - **`llm()` forwards `ctx.signal`** onto `GenerateRequest`; `.stream()` inherits it. -- **`.stream()` is the SAME action**, marked with an ambient sink: it yields UNVALIDATED text and one - final `done` with the value that satisfied `output`; no repair turn (`X_LLM_STREAM_INVALID`); the - budget is reserved before the provider, debited on the first pull, reconciled at `done`; no - `respond` tool; a cache hit yields `done` alone. LAZY. `named()` is re-narrowed so the twin still streams. +- **`.stream()` is the SAME action**, marked with an ambient sink: UNVALIDATED text, then one `done` + with the value that satisfied `output`; no repair turn (`X_LLM_STREAM_INVALID`), no `respond` tool; + budget reserved before the provider, reconciled at `done`; a cache hit yields `done` alone. LAZY. - **`configureAi({ redact })` is the one seam between `vars()` and the provider**; a `Secret` in `vars()` is `X_AI_PROMPT_SECRET` regardless. The `llm.redacted` span attribute records it. - **Fallback is across PROVIDERS serving one model, never across models**; `GenerateResult.provider` @@ -145,7 +141,7 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe each hold half) — never deletes, never touches a template-only closer. The fences are read once at `definePrompt`; no hash moves. - `definePrompt` refuses a re-registered version whose hash moved; the hash is over core's - `canonicalJson` (`prompt.test.ts` pins one hash literally); an absent schema hashes as `''`. + `canonicalJson`; an absent schema hashes as `''`. ## Invariants — providers and the gateway @@ -163,15 +159,11 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe `AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Every other code stays UNREGISTERED on purpose. - **The retry SCHEDULE is core's (`backoffDelay`, `isRetryableStatus`); the LOOP is this package's** — - core's `retry()` retries unclassified throws, and every value here is an app `Provider`'s plain - object, so it would retry a 400. `gateway-backoff.test.ts`. `RetryPolicy`/`DEFAULT_GATEWAY_RETRY` keep the - field names an app writes. + core's `retry()` retries unclassified throws, so it would retry a 400 (`gateway-backoff.test.ts`). - Every non-2xx and every in-band `error` frame is `AiTransportError` with a real `status`. - **Both wire formats answer the same question the same way** (`provider-parity.test.ts`): an in-band - `error` in a 200 body throws (`throwInBandError`, `wire.ts`); a `refusal` stop detail forces - `stopReason: 'refusal'`; a tool call's `input` is parsed (`asToolInput`); the credential is scrubbed - from `detail` (`error-body.ts`'s `withoutKey`/`detailOf`). The status tables and the OpenAI - `estimatedUsage` fallback are asserted as differences. + `error` in a 200 throws (`throwInBandError`); a `refusal` detail forces `stopReason: 'refusal'`; a + tool call's `input` is parsed (`asToolInput`); the credential is scrubbed from `detail`. - A stream ending without `message_stop` throws. **`readSse` caps the unterminated buffer** (`MAX_FRAME_CHARS`); `provider` is required so a transport error names its endpoint. - A tool call is emitted whole. Thinking chunks are never appended to `text`. @@ -188,11 +180,9 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe the body builders, before a socket or a reservation. **Re-registering REPLACES the spec and keeps its rung** (negotiated rates). **Registration order is the ladder within a `family`**, read only by `moreCapableThan`, over the APP's rows; `X_LLM_REFUSED`'s fix names a rung ABOVE or nothing. -- **Every provider serves the app's list**: `anthropicProvider({ models })` and - `openAiProvider({ models })` require a non-empty one (`X_AI_REQUEST_INVALID`, a missing config - too); `echoProvider().models` reads the registry. **No provider has a model of its own**: a direct - call naming none is `X_AI_MODEL_UNRESOLVED` from all three — the list is what it serves, never a - fourth place (`vendor-neutral.test.ts`, one parity case). +- **Every provider serves the app's list**: `anthropicProvider({ models })` / `openAiProvider({ models })` + require a non-empty one (`X_AI_REQUEST_INVALID`); `echoProvider().models` reads the registry. **No + provider has a model of its own**: a direct call naming none is `X_AI_MODEL_UNRESOLVED` from all three. - **The reasoning controls are PER MODEL (`models.ts`)**: an unasked control is omitted; an asked control the model lacks is `X_AI_REQUEST_INVALID`, never dropped. Adding a model is a row the app registers. `disableThinkingUpTo: 'never'` refuses `thinking: 'disabled'` at every effort; @@ -228,7 +218,8 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe - **`agent()` returns an action** (listed in `PRIMITIVE_FACTORIES`). `ctx.actor` is read once and is the only identity any tool runs as — never taken from the model's output. Bounded by `maxTurns` - (`X_AGENT_MAX_TURNS`), `budget.tokensPerRun` and `maxToolResultChars`. + (`X_AGENT_MAX_TURNS`), `budget.tokensPerRun`, `maxToolResultChars`, `maxRepeatedToolCalls` + (`X_AGENT_TOOL_LOOP`) and `deadlineMs` (`X_AGENT_DEADLINE`). - `tools` takes a real `action()` (`AgentTool = AnyAction | ProjectableAction`, `asProjectableAction`); projection happens on the FIRST RUN, memoised. An agent can be another agent's tool. @@ -249,12 +240,11 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe rule a hive member's `reason` follows too. Never a second copy. - **`hive()` is a fan-out action** (`PRIMITIVE_FACTORIES`): `HiveResult` is a SCHEMA from the member's `output`; three arms (`ok`/`failed`/`skipped`) and three counters; `members` in SPLIT - order with `index`; the hive never names an actor; no hive budget code (the reservation holds - it); a member's throw is recorded through `discloseFailure` — a 5xx cause or an uncoded throw's - message becomes `HIDDEN_MEMBER_CAUSE`, logged as facts only; `skipped` reasons - `SKIPPED_ABORTED` / `SKIPPED_NO_INPUT`; `onMemberError` is required; `concurrency` 4, `minMembers` - 2 (a below-floor split still runs serially); an empty split is `X_HIVE_EMPTY`; an aborted `ctx` - unwinds with `X_ABORTED`. + order with `index`; the hive never names an actor; no hive budget code; a member's throw goes + through `discloseFailure` (`HIDDEN_MEMBER_CAUSE`); + `skipped` reasons `SKIPPED_ABORTED` / `SKIPPED_NO_INPUT`; `onMemberError` is required; + `concurrency` 4, `minMembers` 2 (a below-floor split runs serially); an empty split is + `X_HIVE_EMPTY`; an aborted `ctx` is `X_ABORTED`. - **`describeAgents()` / `registeredModels()` are offered, not published** — their only legal consumer is `@ultimat3/cli`. The facts are a THUNK; an unnamed agent has no row. - **`agentJob()` composes `job()`** (never an imitation). `name`, `tenant`, `retry` are @@ -263,16 +253,28 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe idempotent (unenforceable). **`actor`** re-resolves who it acts FOR each attempt (a served worker: ANONYMOUS) via `impersonate`; off-tenant is TERMINAL `X_JOB_TENANT_MISMATCH`. +## Invariants — provider prompt caching + +- **`GenerateRequest.promptCache` is the one switch; what it sends is a ROW in `PROMPT_CACHE_RULES`** + (wire, host, model prefix → marks, session carrier), never an `if`. An unnamed endpoint gets + NOTHING. At most 3 `cache_control` (the vendor's cap is 4); never a `provider` routing object. +- **`agent()` defaults `'auto'`, `llm()` `'off'`.** The stable system part (`prompt.system` + + `system().stable`) is sent FIRST and byte-stable; the mark follows the transcript's tail. The hint + is not in `promptCacheKey`. +- **Usage**: chat-completions `prompt_tokens` INCLUDES read and written tokens; both are subtracted, + clamped. A budget debits all four counters; `estimateCost` prices a hinted prompt as a WRITE. +- **`maxRepeatedToolCalls` compares calls AND results**; `deadlineMs` has no default and rides the + request's signal; a caller abort mid-call is `X_ABORTED`, never the transport's code. + ## Invariants — evals, retrieval, fix lines - **No fix line may name `x ai`** (planned). An eval is selected with `x test eval --filter `; `eval-errors.test.ts` asserts every `x test ` is one of the six types. - Every eval result carries the prompt hash. An eval gates on the DROP from its recorded baseline, - mean AND per case, both sides rounded as the baseline is written. Never-recorded is `X_EVAL_BASELINE_MISSING`, corrupt is `X_EVAL_BASELINE_INVALID`. - `baseline` is `import.meta.resolve('./…')`. Every prompt is named by an eval (`X_EVAL_MISSING`, by - prompt ID). `ULTIMATE_EVAL_RECORD=1` records; a deliberately-worse test calls `run`, never - `assert`; `x verify` with the flag is `X_EVAL_RECORDING` and runs no eval. The gate asks for a - recorded baseline, not only a declaration. + mean AND per case, both sides rounded as the baseline is written (`X_EVAL_BASELINE_MISSING` / + `X_EVAL_BASELINE_INVALID`). `baseline` is `import.meta.resolve('./…')`. Every prompt is named by an + eval (`X_EVAL_MISSING`). `ULTIMATE_EVAL_RECORD=1` records; a deliberately-worse test calls `run`, + never `assert`; `x verify` with the flag is `X_EVAL_RECORDING` and runs no eval. - Retrieval is hybrid by default; no vector-only path. **`chunk()` hard-wraps**, the overlap carry stops at `buffer.length - 1` and yields to a next unit it does not fit beside (no chunk over `size`); `metadata.source` is written AFTER the caller's spread. `VectorStore.prune` is required. diff --git a/packages/ai/README.md b/packages/ai/README.md index 60ba50fe8..35ac613b4 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -435,10 +435,70 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug | `onTurn` reports each completed turn as it happens (and an `agent.turn` span event, always) | a 90-second run emitted nothing until it returned. Observation only — it cannot steer the loop, see the transcript or reach the actor — and a throw from it fails the run rather than being swallowed | | a tool that is not `mcp: { expose: true }` is `X_AGENT_TOOL_UNEXPOSED` **at declaration** | a silently dropped tool reads as offered and is not; `isMcpExposed` is the one predicate, so an in-app agent and an external MCP client see the same catalogue | | running out of turns is `X_AGENT_MAX_TURNS`, never a partial answer | a half-finished transcript returned as a result is working notes presented as a decision | +| `maxRepeatedToolCalls` (default 3) consecutive turns with the same calls **and** results is `X_AGENT_TOOL_LOOP` | such a turn can only be followed by another like it; a polled tool whose answer changes is left alone | +| `deadlineMs` (no default) is the run's wall-clock ceiling: `X_AGENT_DEADLINE` | the in-flight model call is aborted on the socket; a caller that leaves first is still `X_ABORTED` | +| every turn asks the provider to cache its prefix (`promptCache`, default `'auto'`) | the transcript IS the request, so an uncached run pays for its stable prefix once per turn — [below](#provider-prompt-caching) | | `budget.tokensPerRun` caps the **whole run** | a single call is bounded by `maxTokens`; a loop is bounded by nothing until this is set | | a tool result is truncated, and says so | the transcript IS the request, so an untruncated result is re-billed once per remaining turn | | **no semantic cache** | similar prompts do not have similar answers once the answer depends on what `lookupOrder` returned this second | +### Provider prompt caching + +Not the gateway's response cache and not `llm()`'s semantic one: this asks the **provider** to keep +a request's prefix, so the next request reads it at the cache rate. + +```ts +import { agent, definePrompt, t } from '@ultimat3/ai'; +import { can } from '@ultimat3/policy'; + +const kbPrompt = definePrompt<{ question: string }>({ + id: 'kb', + version: '1.0.0', + system: 'Answer from the knowledge base only.', // starts the stable block + template: 'Question: {{question}}', +}); + +export const kbAgent = agent({ + input: t.object({ question: t.string, index: t.string }), + output: t.object({ answer: t.string }), + prompt: kbPrompt, + vars: ({ input }) => ({ question: input.question }), + // `stable` joins prompt.system in the block every run shares; `run` follows it + system: ({ input }) => ({ stable: input.index, run: `Question length: ${input.question.length}` }), + promptCache: { session: 'kb-agent' }, // 'auto' (default) | 'off' | { session } + tools: [], + policy: can('kb:ask'), + onTurn: ({ usage, run }) => { + console.log(usage.cacheReadTokens, usage.cacheWriteTokens, run.cost.minor); + }, +}); +``` + +| Shape | Meaning | +|---|---| +| `GenerateRequest.system: string \| { stable, run? }` | `stable` is byte-identical across runs and is sent first, as its own block; `run` follows it | +| `GenerateRequest.promptCache?: { session? }` | present: mark the stable system part and the transcript's tail, per `PROMPT_CACHE_RULES`. Absent: nothing is marked | +| `agent({ promptCache })` | default `'auto'` — every turn carries the hint | +| `llm({ promptCache })` | default `'off'` — a one-shot call would pay the write premium for a cache nothing reads | +| `AgentTurn.usage` / `AgentTurn.run` | per turn, and the running total: `cacheReadTokens`, `cacheWriteTokens`, and `run.cost` priced once from the summed usage | + +`PROMPT_CACHE_RULES` (`src/prompt-cache.ts`, sources quoted in its header), first match wins: + +| Wire | Host | Model prefix | Marks | `session` | +|---|---|---|---|---| +| Anthropic Messages | any | any | stable system block + last block of the last two messages (3 of 4 breakpoints) | — | +| chat completions | `openrouter.ai` | `anthropic/` | stable system part + the tail message's last part (2) | `x-session-id` header | +| chat completions | `openrouter.ai` | any other | none (implicit caching; a mark on Gemini is a paid write) | `x-session-id` header | +| chat completions | `api.openai.com` | any | none (automatic) | `prompt_cache_key` | +| chat completions | anything else | any | none | — | + +Usage: the chat-completions reader takes `prompt_tokens_details.cached_tokens` and +`.cache_write_tokens` out of `prompt_tokens` (which includes both) and reports them as +`cacheReadTokens` / `cacheWriteTokens`; a missing field is zero. `costOf` prices them at the model +row's `cacheReadPerMillion` / `cacheWritePerMillion` (absent: 0.1x / 1.25x input). A token budget +counts all four counters, one token each; `estimateCost` prices a hinted request's prompt as a +write. No `provider` routing object and no `usage: { include }` is ever sent. + ## `hive()` — many members, one action Fan an action out over many inputs. A factory over a primitive, like `llm()`, `backfill()` and @@ -883,6 +943,8 @@ fail-closed `terminal` default. | `X_LLM_STREAM_INVALID` | a streamed answer failed its schema, and a stream cannot take a repair turn | | `X_AGENT_MAX_TURNS` | an `agent()` used every turn without answering | | `X_AGENT_TOOL_UNEXPOSED` | an `agent()` lists an action no MCP surface exposes | +| `X_AGENT_TOOL_LOOP` | an `agent()` made the same tool calls and read the same results `maxRepeatedToolCalls` turns in a row | +| `X_AGENT_DEADLINE` | an `agent()` outlived its declared `deadlineMs` | | `X_EVAL_THRESHOLD` | an eval scored below its bar | | `X_VECTOR_DIM_MISMATCH` | a vector's length disagrees with the store | | `X_VECTOR_SCOPE_WIDENED` | a derived vector scope tried to leave the tenant it was bound to | @@ -896,7 +958,9 @@ a job boundary the class is gone and the `code` is what survives — match on th | Class | Code | Declared in | |---|---|---| +| `AgentDeadlineError` | `X_AGENT_DEADLINE` | `src/agent-errors.ts` | | `AgentMaxTurnsError` | `X_AGENT_MAX_TURNS` | `src/errors.ts` | +| `AgentToolLoopError` | `X_AGENT_TOOL_LOOP` | `src/agent-errors.ts` | | `AgentToolUnexposedError` | `X_AGENT_TOOL_UNEXPOSED` | `src/errors.ts` | | `AiBudgetExceededError` | `X_AI_BUDGET_EXCEEDED` | `src/errors.ts` | | `AiContentUnsupportedError` | `X_AI_CONTENT_UNSUPPORTED` | `src/content-errors.ts` | diff --git a/packages/ai/src/agent-cache.test.ts b/packages/ai/src/agent-cache.test.ts new file mode 100644 index 000000000..4ea7c3a6d --- /dev/null +++ b/packages/ai/src/agent-cache.test.ts @@ -0,0 +1,302 @@ +/** + * Provider prompt caching through the loop: what `agent()` asks every turn's request to cache, and + * the two properties a cache hit depends on — a stable block that is the same BYTES across turns + * and runs, and a mark that follows the transcript's tail. + */ + +import { describe, expect, test } from 'bun:test'; +import { ctxOf, userActor } from '@ultimat3/core'; +import { allow } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import type { AgentTurn } from './agent'; +import { agent } from './agent'; +import { echoProvider } from './echo-provider'; +import { providerGateway } from './gateway'; +import { registerModel } from './models'; +import { chatCompletionBody } from './openai-body'; +import { definePrompt } from './prompt'; +import type { GenerateRequest, GenerateResult, Provider, TokenUsage } from './provider'; +import { anthropicProvider, costOf } from './provider'; +import { configureAi, resetAiRuntime } from './runtime'; +import type { ProjectableAction } from './tools'; + +const MODEL = 'anthropic/claude-sonnet-5.5'; +const usd = (minor: number) => ({ minor, currency: 'USD' }); + +/** OpenRouter's list prices for the id above, 2026-10-11: $2 in, $10 out, $0.10 read, $2.50 write. */ +function register(): void { + registerModel({ + id: MODEL, + contextWindow: 1_000_000, + maxOutput: 128_000, + inputPerMillion: usd(200), + outputPerMillion: usd(1_000), + cacheReadPerMillion: usd(10), + cacheWritePerMillion: usd(250), + cacheMinimumTokens: 512, + reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, + }); +} + +const Input = t.object({ question: t.string }); +const Output = t.object({ answer: t.string }); +const RULES = 'RULES: answer from the index only.'; +const INDEX = `INDEX\n${Array.from({ length: 40 }, (_, n) => `doc-${n}: title ${n}`).join('\n')}`; +const MARK = { type: 'ephemeral' }; + +/** Two tool turns, then an answer — with the usage a caching provider reports for each. */ +const USAGES: readonly TokenUsage[] = [ + { inputTokens: 20, outputTokens: 30, cacheReadTokens: 0, cacheWriteTokens: 6_000 }, + { inputTokens: 10, outputTokens: 30, cacheReadTokens: 6_000, cacheWriteTokens: 300 }, + { inputTokens: 10, outputTokens: 40, cacheReadTokens: 6_300, cacheWriteTokens: 300 }, +]; + +function scripted(): { provider: Provider; seen: GenerateRequest[] } { + const seen: GenerateRequest[] = []; + const provider: Provider = { + name: 'scripted', + models: [MODEL], + generate(request) { + const turn = seen.length % USAGES.length; + seen.push(request); + const usage = USAGES[turn] ?? expect.unreachable('no usage scripted'); + const call = + turn < 2 + ? { id: `call-${seen.length}`, name: 'readDoc', input: { n: turn } } + : { id: `call-${seen.length}`, name: 'respond', input: { answer: 'done' } }; + return Promise.resolve({ + model: MODEL, + text: '', + toolCalls: [call], + stopReason: 'tool_use', + stopDetails: undefined, + usage, + cost: costOf(MODEL, usage), + } satisfies GenerateResult); + }, + stream: (request) => echoProvider().stream(request), + }; + return { provider, seen }; +} + +const readDoc: ProjectableAction = { + name: 'readDoc', + description: 'read one document', + mcp: { expose: true }, + run: ({ input }) => Promise.resolve(`body of ${JSON.stringify(input)}`), +}; + +let seq = 0; +function declare( + extra: Partial< + Parameters>[0] + > = {}, + turns: AgentTurn[] = [], + dynamic = true, +) { + seq += 1; + return agent({ + input: Input, + output: Output, + prompt: definePrompt<{ question: string }>({ + id: `kb-${seq}`, + version: '1.0.0', + system: RULES, + template: 'Question: {{question}}', + }), + vars: ({ input }) => ({ question: input.question }), + ...(dynamic + ? { + system: ({ input }: { input: { question: string } }) => ({ + stable: INDEX, + run: `Asked: ${input.question.length} chars`, + }), + } + : {}), + tools: [readDoc], + policy: allow(), + onTurn: (turn) => void turns.push(turn), + ...extra, + }).named(`kbAgent${seq}`); +} + +function install(): GenerateRequest[] { + resetAiRuntime(); + register(); + const { provider, seen } = scripted(); + configureAi({ gateway: providerGateway({ defaultModel: MODEL, providers: [provider] }) }); + return seen; +} + +const ctx = () => ctxOf({ actor: userActor({ id: 'u-1', orgId: 'org-1' }) }); + +/** The request as the chat-completions wire to OpenRouter would carry it. */ +const wire = (request: GenerateRequest) => + chatCompletionBody({ + request, + model: MODEL, + provider: 'openrouter', + stream: false, + baseUrl: 'https://openrouter.ai/api/v1', + })['messages'] as { role: string; content: unknown }[]; + +/** The first content part of a message whose content is parts. */ +const firstPart = (message: { content: unknown } | undefined): unknown => + Array.isArray(message?.content) ? message.content[0] : undefined; + +/** Indexes of the messages whose content carries a mark. */ +const markedAt = (messages: readonly unknown[]): number[] => + messages.flatMap((message, index) => + JSON.stringify(message).includes('cache_control') ? [index] : [], + ); + +describe('agent() caches its prefix by default', () => { + test('every turn carries the hint; the stable block is byte-identical across turns and runs', async () => { + const seen = install(); + const kb = declare(); + await kb({ question: 'first?' }, { ctx: ctx() }); + await kb({ question: 'a different, longer question?' }, { ctx: ctx() }); + + expect(seen).toHaveLength(6); + for (const request of seen) expect(request.promptCache).toEqual({}); + const stable = seen.map((request) => + typeof request.system === 'object' ? request.system.stable : undefined, + ); + // The prompt's static system and `system().stable`, as ONE block. + expect(stable[0]).toBe(`${RULES}\n\n${INDEX}`); + expect(new Set(stable).size).toBe(1); + // …and the same bytes on the wire: the first system part of all six requests. + const first = seen.map((request) => JSON.stringify(firstPart(wire(request)[0]))); + expect(new Set(first).size).toBe(1); + expect(JSON.parse(first[0] ?? '{}')).toEqual({ + type: 'text', + text: `${RULES}\n\n${INDEX}`, + cache_control: MARK, + }); + // The per-run part differs between the runs and sits AFTER the stable one. + const run = seen.map((request) => + typeof request.system === 'object' ? request.system.run : undefined, + ); + expect(run[0]).toBe('Asked: 6 chars'); + expect(run[3]).toBe('Asked: 29 chars'); + }); + + test('the mark moves to the new tail each turn, and turn N+1 still contains turn N’s prefix', async () => { + const seen = install(); + await declare()({ question: 'q' }, { ctx: ctx() }); + const [one, two, three] = seen.map(wire); + if (one === undefined || two === undefined || three === undefined) { + return expect.unreachable('three turns were scripted'); + } + // system + the tail, and only those: [system, user] → [.., assistant, tool] → [.., assistant, tool] + expect(markedAt(one)).toEqual([0, 1]); + expect(markedAt(two)).toEqual([0, 3]); + expect(markedAt(three)).toEqual([0, 5]); + // Apart from the mark itself, each request is a strict extension of the one before: the + // prefix turn N wrote is the prefix turn N+1 sends. + const unmarked = (messages: unknown) => + JSON.parse(JSON.stringify(messages), (key, value) => + key === 'cache_control' ? undefined : value, + ) as { content: unknown }[]; + const text = (message: { content: unknown } | undefined) => + JSON.stringify( + Array.isArray(message?.content) && message.content.length === 1 + ? (message.content[0] as { text?: string }).text + : message?.content, + ); + expect(unmarked(two).slice(0, 2).map(text)).toEqual(unmarked(one).map(text)); + expect(unmarked(three).slice(0, 4).map(text)).toEqual(unmarked(two).map(text)); + }); + + test('the same transcript on the Anthropic wire never exceeds four breakpoints', async () => { + const seen = install(); + await declare()({ question: 'q' }, { ctx: ctx() }); + registerModel({ + id: 'claude-sonnet-5-5', + contextWindow: 1_000_000, + maxOutput: 128_000, + inputPerMillion: usd(200), + outputPerMillion: usd(1_000), + cacheMinimumTokens: 512, + reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, + }); + const native = anthropicProvider({ models: ['claude-sonnet-5-5'], apiKey: 'k' }); + for (const request of seen) { + const body = JSON.stringify(native.body({ ...request, model: 'claude-sonnet-5-5' })); + const count = body.split('"cache_control"').length - 1; + expect(count).toBeGreaterThanOrEqual(2); + expect(count).toBeLessThanOrEqual(4); + } + }); + + test("promptCache: 'off' sends no hint; a session key is carried on every turn", async () => { + const seen = install(); + await declare({ promptCache: 'off' })({ question: 'q' }, { ctx: ctx() }); + for (const request of seen) expect(request.promptCache).toBeUndefined(); + expect(markedAt(wire(seen[2] ?? expect.unreachable('three turns')))).toEqual([]); + + const keyed = install(); + await declare({ + promptCache: { + session: ({ ctx: c }) => `kb:${c.actor.kind === 'user' ? c.actor.orgId : ''}`, + }, + })({ question: 'q' }, { ctx: ctx() }); + expect(keyed.map((request) => request.promptCache?.session)).toEqual([ + 'kb:org-1', + 'kb:org-1', + 'kb:org-1', + ]); + }); + + test('an agent with no system() and no per-run part still sends its system as one string', async () => { + const seen = install(); + await declare({}, [], false)({ question: 'q' }, { ctx: ctx() }); + expect(seen[0]?.system).toBe(RULES); + }); +}); + +describe('usage and cost, per turn and per run', () => { + test('onTurn reports cache reads and writes, priced at the cache rates, and the running total', async () => { + install(); + const turns: AgentTurn[] = []; + await declare({}, turns)({ question: 'q' }, { ctx: ctx() }); + + expect(turns.map((turn) => turn.usage.cacheWriteTokens)).toEqual([6_000, 300, 300]); + expect(turns.map((turn) => turn.usage.cacheReadTokens)).toEqual([0, 6_000, 6_300]); + // Turn 1: 20 in @200 + 6,000 written @250 + 30 out @1,000 per million = 0.004 + 1.5 + 0.03 + // minor units of input side → ceil(1.504) + ceil(0.03) = 2 + 1. + expect(turns[0]?.cost).toEqual(usd(3)); + // Turn 2: 10 in + 6,000 READ @10 + 300 written = 0.002 + 0.06 + 0.075 → 1, plus 1 for output. + expect(turns[1]?.cost).toEqual(usd(2)); + // The run: summed usage, priced once — never three round-ups added together. + const last = turns.at(-1)?.run; + expect(last?.usage).toEqual({ + inputTokens: 40, + outputTokens: 100, + cacheReadTokens: 12_300, + cacheWriteTokens: 6_600, + }); + // 40@200 + 12,300@10 + 6,600@250 = 0.008 + 0.123 + 1.65 = 1.781 → 2; output 0.1 → 1. + expect(last?.cost).toEqual(usd(3)); + expect(turns[0]?.run.usage).toEqual(turns[0]?.usage as TokenUsage); + }); + + test('the same run without a cache would have cost the prefix at the input rate every turn', () => { + register(); + const cached = costOf(MODEL, { + inputTokens: 40, + outputTokens: 0, + cacheReadTokens: 12_300_000, + cacheWriteTokens: 6_600_000, + }); + const uncached = costOf(MODEL, { + inputTokens: 18_900_040, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }); + // 12.3M read @10 + 6.6M written @250 = 123 + 1,650; against 18.9M @200 = 3,780. + expect(cached.minor).toBe(1_774); + expect(uncached.minor).toBe(3_781); + }); +}); diff --git a/packages/ai/src/agent-def.ts b/packages/ai/src/agent-def.ts new file mode 100644 index 000000000..62087f3cd --- /dev/null +++ b/packages/ai/src/agent-def.ts @@ -0,0 +1,111 @@ +// Single responsibility: what an `agent()` DECLARATION is — its options, and the one event its +// observer is handed. Types only; ./agent owns the loop that reads them. + +import type { ActionMcp, ActionPolicy } from '@ultimat3/action'; +import type { Ctx } from '@ultimat3/core'; +import type { Money } from '@ultimat3/money'; +import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema'; +import type { AgentSystem } from './agent-system'; +import type { AgentRunUsage } from './agent-usage'; +import type { LlmBudget } from './llm'; +import type { ModelId } from './models'; +import type { Prompt, PromptVars } from './prompt'; +import type { PromptCacheOption } from './prompt-cache'; +import type { StopReason, TokenUsage } from './provider'; +import type { AgentTool } from './tools'; + +export interface AgentBudget extends LlmBudget { + /** + * Token ceiling for the WHOLE run, every turn counted. The one ceiling `llm()` does not need: + * a single call is bounded by `maxTokens`, a loop is bounded by nothing until this is set. + */ + readonly tokensPerRun?: number; +} + +export interface AgentVarsArgs { + readonly input: InferOutput; + readonly ctx: Ctx; +} + +export interface AgentDef< + TInput extends StandardSchemaV1, + TOutput extends StandardSchemaV1, + V extends PromptVars, +> { + readonly model?: ModelId; + readonly input: TInput; + readonly output: TOutput; + readonly prompt: Prompt; + /** Same contract as `llm()`: the one declared place a run loads data. */ + vars(args: AgentVarsArgs): V | Promise; + /** + * System text loaded at RUN time, beside the prompt's static `system`: a knowledge-base index, + * today's date. Redacted like the prompt. Not part of the prompt's hash, exactly as `vars` is not. + */ + system?(args: AgentVarsArgs): AgentSystem | Promise; + /** + * Provider prompt caching. Default `'auto'`: every turn's request asks the provider to cache its + * prefix (the stable system block, then the transcript up to its tail), so turn N+1 reads turn + * N's prefix at the cache rate instead of paying input for it again. `'off'` sends no mark. + * `{ session }` is `'auto'` plus a key for hosts that route by one. `PROMPT_CACHE_RULES` decides + * what each wire is sent; a host that caches implicitly is sent nothing either way. + */ + readonly promptCache?: PromptCacheOption>; + /** + * The actions the model may call. Each must be `mcp: { expose: true }` — the same predicate an + * external MCP client is filtered by, so an in-app agent and an external one are offered + * exactly the same tools. Listing one that is not exposed is refused at declaration rather than + * dropped, because a tool that reads as offered and silently is not is the worst of both. + */ + readonly tools: readonly AgentTool[]; + /** Hard ceiling on model turns. Reaching it is `X_AGENT_MAX_TURNS`, never a partial answer. */ + readonly maxTurns?: number; + readonly maxToolResultChars?: number; + /** + * Consecutive turns that may make the SAME tool calls and read the SAME results. Reaching it is + * `X_AGENT_TOOL_LOOP`. Default 3. + */ + readonly maxRepeatedToolCalls?: number; + /** + * Wall-clock ceiling for the whole run, in milliseconds. Passing it aborts the model call in + * flight and is `X_AGENT_DEADLINE`. No default: a timeout nobody declared is a lost answer. + */ + readonly deadlineMs?: number; + readonly budget?: AgentBudget; + readonly policy: ActionPolicy; + readonly mcp?: ActionMcp; + /** Enforced completion ceiling PER TURN. The model never sees it. */ + readonly maxTokens?: number; + /** + * Called once per completed model turn, before the answer or the tool calls are acted on. The + * one thing a multi-turn run could not do until 2026-08: a 90-second loop emitted nothing until + * it returned, so a progress indicator, a per-turn spend line and a transcript log all had to be + * hand-rolled outside the framework — which is exactly the loop `agent()` exists to replace. + * + * Observation only: it cannot steer the loop, cannot see the transcript and cannot reach the + * actor. A throw from it FAILS the run rather than being swallowed — it is the app's code on the + * run's own path, and an observer that silently stopped working would be indistinguishable from + * one that is fine. The same facts land on the span as an `agent.turn` event, so a run that + * declares no hook is still readable in a trace. + * + * NOT `.stream()`. Tokens on a screen is a different contract, and `llm()`'s streamed half is + * one turn deep by construction; this is per TURN. + */ + onTurn?(event: AgentTurn): void | Promise; +} + +/** One completed model turn, as an observer sees it. Facts only — no transcript, no actor. */ +export interface AgentTurn { + /** 1-based, so `turn === maxTurns` is the last one this run will take. */ + readonly turn: number; + readonly maxTurns: number; + readonly model: ModelId; + /** Tools this turn asked for, in the order the model emitted them. `respond` is not one. */ + readonly toolCalls: readonly string[]; + readonly stopReason: StopReason; + readonly usage: TokenUsage; + /** This turn's cost alone, integer minor units — never the run's running total. */ + readonly cost: Money; + /** The running total, this turn included — on the last turn, the whole run. */ + readonly run: AgentRunUsage; +} diff --git a/packages/ai/src/agent-errors.ts b/packages/ai/src/agent-errors.ts new file mode 100644 index 000000000..4e1c17d23 --- /dev/null +++ b/packages/ai/src/agent-errors.ts @@ -0,0 +1,42 @@ +// The two X_AGENT_* codes an UNATTENDED run needs, apart from ./errors only because one file has +// one job and that catalogue is at its ceiling — the split `hive-errors.ts` made. The codes, their +// titles and the single `registerErrorCodes` call stay in ./errors. + +import { UltimateError } from '@ultimat3/core'; + +/** + * The loop asked for the same tool calls and read the same results, turn after turn. + * + * Call AND result, never the call alone: polling one tool with one argument is legitimate while + * what comes back keeps changing, and it stops being progress the moment it does not. A turn that + * repeats both can only be followed by a turn that repeats both, so every further turn buys the + * same non-answer over a transcript that is one turn longer. + */ +export class AgentToolLoopError extends UltimateError { + constructor(input: { agent: string; repeats: number; tools: readonly string[] }) { + super({ + code: 'X_AGENT_TOOL_LOOP', + cause: + `agent "${input.agent}" made the same tool calls (${input.tools.join(', ')}) and read the ` + + `same results ${input.repeats} turns in a row`, + fix: 'maxRepeatedToolCalls: 5 # on the agent, ONLY for a tool that is polled and answers identically while it waits — otherwise tell the template what to do when a tool result does not change, then bump its version', + meta: { agent: input.agent, repeats: input.repeats, tools: input.tools }, + }); + } +} + +/** + * The run was still going when its declared `deadlineMs` elapsed. Distinct from `X_ABORTED` — the + * caller did not leave, the declaration ran out — and from `X_AGENT_MAX_TURNS`, because the fix is + * about wall clock: a slow tool or a slow model, not a loop with no exit. + */ +export class AgentDeadlineError extends UltimateError { + constructor(input: { agent: string; deadlineMs: number }) { + super({ + code: 'X_AGENT_DEADLINE', + cause: `agent "${input.agent}" was still running after its ${input.deadlineMs}ms deadline; the model call in flight was aborted and no further turn or tool was started`, + fix: 'deadlineMs: 300_000 # on the agent, only once a trace shows the run converging — otherwise lower maxTurns, or shorten the slow tool the agent.turn span events name', + meta: { agent: input.agent, deadlineMs: input.deadlineMs }, + }); + } +} diff --git a/packages/ai/src/agent-guards.test.ts b/packages/ai/src/agent-guards.test.ts new file mode 100644 index 000000000..67470adba --- /dev/null +++ b/packages/ai/src/agent-guards.test.ts @@ -0,0 +1,221 @@ +/** + * The two guards an unattended run needs: a loop that repeats itself is stopped before it spends + * every turn, and a run that outlives its deadline is stopped whatever it is waiting on. + */ + +import { describe, expect, test } from 'bun:test'; +import { ctxOf, userActor } from '@ultimat3/core'; +import { allow } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { agent } from './agent'; +import { echoProvider } from './echo-provider'; +import { providerGateway } from './gateway'; +import { FIXTURE_MODEL, useFixtureModels } from './model-fixture'; +import { definePrompt } from './prompt'; +import type { GenerateRequest, GenerateResult, Provider, TokenUsage } from './provider'; +import { costOf } from './provider'; +import { configureAi, resetAiRuntime } from './runtime'; +import type { ProjectableAction } from './tools'; + +useFixtureModels(); + +const Input = t.object({ id: t.string }); +const Output = t.object({ answer: t.string }); +const USAGE: TokenUsage = { + inputTokens: 5, + outputTokens: 5, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}; + +type Call = { readonly name: string; readonly input: Record }; + +/** Answers turn N with `script(N)`; `wait` holds every answer until the request's signal aborts. */ +function model(script: (turn: number) => Call, wait = false) { + const seen: GenerateRequest[] = []; + const provider: Provider = { + name: 'scripted', + models: [FIXTURE_MODEL], + generate(request) { + seen.push(request); + const call = script(seen.length); + const result: GenerateResult = { + model: FIXTURE_MODEL, + text: '', + toolCalls: [{ id: `call-${seen.length}`, ...call }], + stopReason: 'tool_use', + stopDetails: undefined, + usage: USAGE, + cost: costOf(FIXTURE_MODEL, USAGE), + }; + if (!wait) return Promise.resolve(result); + // A socket that only ever ends by being aborted — what a hung provider looks like. + return new Promise((_, reject) => { + request.signal?.addEventListener('abort', () => reject(request.signal?.reason)); + }); + }, + stream: (request) => echoProvider().stream(request), + }; + resetAiRuntime(); + configureAi({ gateway: providerGateway({ defaultModel: FIXTURE_MODEL, providers: [provider] }) }); + return seen; +} + +let seq = 0; +function declare( + tools: readonly ProjectableAction[], + extra: { maxRepeatedToolCalls?: number; deadlineMs?: number; maxTurns?: number } = {}, +) { + seq += 1; + return agent({ + input: Input, + output: Output, + prompt: definePrompt<{ id: string }>({ + id: `guard-${seq}`, + version: '1.0.0', + template: 'Resolve {{id}}.', + }), + vars: ({ input }) => ({ id: input.id }), + tools, + maxTurns: 8, + policy: allow(), + ...extra, + }).named(`guardAgent${seq}`); +} + +const tool = (name: string, run: ProjectableAction['run']): ProjectableAction => ({ + name, + description: name, + mcp: { expose: true }, + run, +}); + +const ctx = () => ctxOf({ actor: userActor({ id: 'u-1' }) }); +const status = (): Call => ({ name: 'status', input: { id: 'j-1' } }); + +describe('a loop that repeats itself is stopped', () => { + test('the same call with the same result, three turns running, is X_AGENT_TOOL_LOOP', async () => { + const seen = model(status); + let ran = 0; + const run = declare([ + tool('status', () => { + ran += 1; + return Promise.resolve('pending'); + }), + ]); + const call = run({ id: 'x' }, { ctx: ctx() }); + await expect(call).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + meta: { repeats: 3, tools: ['status'] }, + }); + // Three turns bought, not eight: the fourth request was never sent. + expect(seen).toHaveLength(3); + expect(ran).toBe(3); + }); + + test('a polled tool whose answer keeps changing is progress, and is left alone', async () => { + let ran = 0; + const seen = model((turn) => + turn <= 5 ? status() : { name: 'respond', input: { answer: 'done' } }, + ); + const run = declare([ + tool('status', () => { + ran += 1; + return Promise.resolve(`pending ${ran}`); + }), + ]); + expect(await run({ id: 'x' }, { ctx: ctx() })).toEqual({ answer: 'done' }); + expect(seen).toHaveLength(6); + }); + + test('a different call in between resets the count; key order in the arguments does not', async () => { + // a, a, b, a, a — never three in a row — then answer. + const order = ['a', 'a', 'b', 'a', 'a']; + const mixed = model((turn) => { + const id = order[turn - 1]; + if (id === undefined) return { name: 'respond', input: { answer: 'done' } }; + // The same arguments spelled in two key orders are the same call. + return { name: 'status', input: turn % 2 === 0 ? { id, n: 1 } : { n: 1, id } }; + }); + const run = declare([tool('status', () => Promise.resolve('pending'))]); + expect(await run({ id: 'x' }, { ctx: ctx() })).toEqual({ answer: 'done' }); + expect(mixed).toHaveLength(6); + + const same = model((turn) => ({ + name: 'status', + input: turn % 2 === 0 ? { id: 'a', n: 1 } : { n: 1, id: 'a' }, + })); + const again = declare([tool('status', () => Promise.resolve('pending'))]); + await expect(again({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + }); + expect(same).toHaveLength(3); + }); + + test('the limit is the declaration’s, and a limit that could never pass a first turn is refused', async () => { + const seen = model(status); + const run = declare([tool('status', () => Promise.resolve('pending'))], { + maxRepeatedToolCalls: 5, + }); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + meta: { repeats: 5 }, + }); + expect(seen).toHaveLength(5); + for (const bad of [1, 0, Number.NaN, 2.5]) { + expect(() => declare([], { maxRepeatedToolCalls: bad })).toThrow('maxRepeatedToolCalls'); + } + }); +}); + +describe('a run has a deadline when it declares one', () => { + test('a hung model call is aborted at the deadline and reported as X_AGENT_DEADLINE', async () => { + const seen = model(status, true); + const run = declare([tool('status', () => Promise.resolve('pending'))], { deadlineMs: 30 }); + const started = performance.now(); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_DEADLINE', + meta: { deadlineMs: 30 }, + }); + expect(performance.now() - started).toBeLessThan(1_000); + // The socket itself heard it: the request's own signal is aborted. + expect(seen[0]?.signal?.aborted).toBe(true); + expect(seen).toHaveLength(1); + }); + + test('a tool that never returns does not hold the run past it, and no further turn is bought', async () => { + const seen = model(status); + const run = declare([tool('status', () => new Promise(() => {}))], { deadlineMs: 30 }); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_DEADLINE', + }); + expect(seen).toHaveLength(1); + }); + + test('the caller leaving first is still X_ABORTED, and no deadline means no timer', async () => { + const controller = new AbortController(); + const hung = model(status, true); + const run = declare([tool('status', () => Promise.resolve('pending'))], { deadlineMs: 5_000 }); + const call = run( + { id: 'x' }, + { ctx: ctxOf({ actor: userActor({ id: 'u-1' }), signal: controller.signal }) }, + ); + setTimeout(() => controller.abort(), 10); + // Not the deadline's code, and not the transport's either: the aborted socket surfaces as a + // provider failure, which is retryable — a job would have re-run a run nobody is waiting for. + await expect(call).rejects.toMatchObject({ code: 'X_ABORTED' }); + expect(hung).toHaveLength(1); + + // Undeclared: the request is handed the caller's own signal, untouched. + const seen = model(() => ({ name: 'respond', input: { answer: 'done' } })); + const c = ctx(); + await declare([])({ id: 'x' }, { ctx: c }); + expect(seen[0]?.signal).toBe(c.signal); + }); + + test('a deadline that is not a positive whole number of milliseconds is refused at declaration', () => { + for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => declare([], { deadlineMs: bad })).toThrow('deadlineMs'); + } + }); +}); diff --git a/packages/ai/src/agent-guards.ts b/packages/ai/src/agent-guards.ts new file mode 100644 index 000000000..8ead8576c --- /dev/null +++ b/packages/ai/src/agent-guards.ts @@ -0,0 +1,90 @@ +// Single responsibility: the two guards that let an `agent()` run with nobody watching — the +// repeated-turn detector and the per-run deadline. Apart from ./agent so the loop reads as a loop. + +import { fingerprint } from '@ultimat3/core'; +import { AgentDeadlineError, AgentToolLoopError } from './agent-errors'; +import type { LlmToolCall, LlmToolResult } from './tools'; + +/** + * Consecutive identical turns an agent may take when the declaration omits a number. Three: one + * repeat is a model re-checking, two is a model that has not read the answer, and nothing a third + * identical turn returns can make the fourth differ. + */ +export const DEFAULT_MAX_REPEATED_TOOL_CALLS = 3; + +/** + * Called once per tool turn, with what the model asked for and what it was told. Throws + * `X_AGENT_TOOL_LOOP` on the `limit`-th consecutive turn whose calls AND results are the ones + * before it. A fingerprint, not the text: one result can be megabytes before it is truncated. + */ +export function toolLoopGuard( + agent: string, + limit: number, +): (calls: readonly LlmToolCall[], results: readonly LlmToolResult[]) => void { + let last: string | undefined; + let repeats = 0; + return (calls, results) => { + const shape = fingerprint({ + calls: calls.map((call) => [call.name, call.input]), + results: results.map((result) => [result.content, result.isError === true]), + }); + repeats = shape === last ? repeats + 1 : 1; + last = shape; + if (repeats >= limit) { + throw new AgentToolLoopError({ agent, repeats, tools: calls.map((call) => call.name) }); + } + }; +} + +export interface RunDeadline { + /** The caller's signal and the deadline as one — what every provider call is handed. */ + readonly signal: AbortSignal; + /** Throws `X_AGENT_DEADLINE` once the deadline has passed. Called before each turn and batch. */ + check(): void; + /** + * Await `work`, but no longer than the deadline, and report a failure the deadline caused AS the + * deadline: the aborted socket surfaces as a transport error, which would otherwise read as the + * provider's fault. Work that outlives the race is left to finish — a tool has no signal to hear. + */ + guard(work: Promise): Promise; +} + +/** `ms` undefined is no deadline: the caller's signal, and two no-ops. */ +export function runDeadline( + agent: string, + ms: number | undefined, + caller: AbortSignal, +): RunDeadline { + if (ms === undefined) return { signal: caller, check: () => {}, guard: (work) => work }; + const timer = AbortSignal.timeout(ms); + const expired = (): AgentDeadlineError => new AgentDeadlineError({ agent, deadlineMs: ms }); + // The caller leaving first stays `X_ABORTED`: it is the truer account of why the run stopped. + const byDeadline = (): boolean => timer.aborted && !caller.aborted; + return { + signal: AbortSignal.any([caller, timer]), + check: () => { + if (byDeadline()) throw expired(); + }, + guard: async (work) => { + if (byDeadline()) { + // Abandoned before it was awaited; its own rejection must not go unhandled. + work.catch(() => {}); + throw expired(); + } + let stop = (): void => {}; + const timeout = new Promise((_, reject) => { + stop = () => reject(expired()); + timer.addEventListener('abort', stop, { once: true }); + }); + try { + return await Promise.race([work, timeout]); + } catch (error) { + work.catch(() => {}); + if (byDeadline()) throw expired(); + throw error; + } finally { + timer.removeEventListener('abort', stop); + } + }, + }; +} diff --git a/packages/ai/src/agent-job.test.ts b/packages/ai/src/agent-job.test.ts index d4c314e79..d2bfdacb4 100644 --- a/packages/ai/src/agent-job.test.ts +++ b/packages/ai/src/agent-job.test.ts @@ -109,6 +109,8 @@ function summariser( vars: ({ input }) => ({ topic: input.topic }), tools, ...(budget === undefined ? {} : { budget }), + // A scripted model may repeat one call for ever; the ceilings under test are the others. + maxRepeatedToolCalls: 100, maxTurns: 6, policy: allow(), }).named(name); diff --git a/packages/ai/src/agent-system.ts b/packages/ai/src/agent-system.ts new file mode 100644 index 000000000..17fe0a058 --- /dev/null +++ b/packages/ai/src/agent-system.ts @@ -0,0 +1,37 @@ +// Single responsibility: the system prompt ONE agent run sends — the prompt's static text and what +// the declaration's `system()` loaded, composed into the stable and per-run parts a cache keys on. + +import type { SystemPrompt } from './prompt-cache'; + +/** + * What `system()` adds to the prompt's own `system`. `stable` joins it in the block every run + * shares — it must be BYTE-identical run to run (sorted, undated), or each run pays a full cache + * write for it. `run` is a second block after it, for whatever differs. + */ +export interface AgentSystem { + readonly stable?: string; + readonly run?: string; +} + +/** + * The system prompt one run sends: the prompt's static text and `system()`'s stable part as ONE + * block (the half every run shares), the per-run part after it. A plain string when there is no + * per-run part — the shape every request had before the parts existed. + */ +export function systemOf( + fixed: string | undefined, + extra: AgentSystem, + redact: (text: string) => string, +): SystemPrompt | undefined { + const stable = [fixed ?? '', extra.stable ?? ''] + .filter((part) => part !== '') + .map(redact) + .join('\n\n'); + const run = extra.run === undefined || extra.run === '' ? '' : redact(extra.run); + if (run !== '') return { stable, run }; + return fixed === undefined && stable === '' ? undefined : stable; +} + +/** Whether two composed prompts are the same text — what `llm.redacted` compares. */ +export const sameSystem = (a: SystemPrompt | undefined, b: SystemPrompt | undefined): boolean => + JSON.stringify(a) === JSON.stringify(b); diff --git a/packages/ai/src/agent-usage.ts b/packages/ai/src/agent-usage.ts new file mode 100644 index 000000000..d92c5c1c1 --- /dev/null +++ b/packages/ai/src/agent-usage.ts @@ -0,0 +1,27 @@ +// Single responsibility: a run's running token and money total, across its turns. + +import type { Money } from '@ultimat3/money'; +import type { GenerateResult, TokenUsage } from './provider'; +import { costOf } from './provider'; + +/** The run so far, this turn included. */ +export interface AgentRunUsage { + readonly usage: TokenUsage; + readonly cost: Money; +} + +/** + * The run's total after one more turn. The cost is re-priced from the summed usage rather than + * summed per turn: each turn's cost is rounded UP to a minor unit, and ten round-ups is not the + * price of the run. + */ +export function addUsage(total: AgentRunUsage | undefined, result: GenerateResult): AgentRunUsage { + const before = total?.usage; + const usage: TokenUsage = { + inputTokens: (before?.inputTokens ?? 0) + result.usage.inputTokens, + outputTokens: (before?.outputTokens ?? 0) + result.usage.outputTokens, + cacheReadTokens: (before?.cacheReadTokens ?? 0) + result.usage.cacheReadTokens, + cacheWriteTokens: (before?.cacheWriteTokens ?? 0) + result.usage.cacheWriteTokens, + }; + return { usage, cost: costOf(result.model, usage) }; +} diff --git a/packages/ai/src/agent.test.ts b/packages/ai/src/agent.test.ts index 464b01d0f..fe484d6a6 100644 --- a/packages/ai/src/agent.test.ts +++ b/packages/ai/src/agent.test.ts @@ -216,6 +216,9 @@ describe('the loop is bounded', () => { prompt: promptFor(), vars: ({ input }) => ({ orderId: input.orderId }), tools: [actorProbe([])], + // The scripted model repeats one call for ever; the repeated-turn guard is lifted so the + // ceiling under test is the one that stops it. + maxRepeatedToolCalls: 100, maxTurns: 3, policy: allow(), }).named('loopingAgent'); @@ -237,6 +240,9 @@ describe('the loop is bounded', () => { prompt: promptFor(), vars: ({ input }) => ({ orderId: input.orderId }), tools: [actorProbe([])], + // The scripted model repeats one call for ever; the repeated-turn guard is lifted so the + // ceiling under test is the one that stops it. + maxRepeatedToolCalls: 100, maxTurns: 10, // Enough for the first turn's estimate, not for a second. budget: { tokensPerRun: 4_200 }, diff --git a/packages/ai/src/agent.ts b/packages/ai/src/agent.ts index 43a2c58b1..608a77d84 100644 --- a/packages/ai/src/agent.ts +++ b/packages/ai/src/agent.ts @@ -20,38 +20,36 @@ * one run's world state to another. Version bump plus the tools' own caching is the story. */ -import type { Action, ActionMcp, ActionPolicy } from '@ultimat3/action'; +import type { Action } from '@ultimat3/action'; import { action } from '@ultimat3/action'; import type { Ctx, Span } from '@ultimat3/core'; import { finiteCount, isMcpExposed, throwIfAborted, withSpan } from '@ultimat3/core'; -import type { Money } from '@ultimat3/money'; import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema'; import { formatIssues, validateAsync } from '@ultimat3/schema'; +import type { AgentDef, AgentTurn } from './agent-def'; import type { AgentFact } from './agent-facts'; import { registerAgentFact } from './agent-facts'; +import { DEFAULT_MAX_REPEATED_TOOL_CALLS, runDeadline, toolLoopGuard } from './agent-guards'; +import { sameSystem, systemOf } from './agent-system'; import { assistantTurn, repairTurn, toolResultTurn } from './agent-transcript'; +import type { AgentRunUsage } from './agent-usage'; +import { addUsage } from './agent-usage'; import type { BudgetLimits } from './budget'; import { budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget'; import { AgentMaxTurnsError, AgentToolUnexposedError, + AiRequestInvalidError, LlmOutputInvalidError, LlmRefusedError, LlmTruncatedError, } from './errors'; -import type { LlmBudget } from './llm'; import { answerAttributes } from './llm'; import { describedModel, type ModelSource, modelForCall } from './model-resolve'; -import type { ModelId } from './models'; import { moreCapableThan } from './models'; -import type { Prompt, PromptVars } from './prompt'; -import type { - AiMessage, - GenerateRequest, - GenerateResult, - StopReason, - TokenUsage, -} from './provider'; +import type { PromptVars } from './prompt'; +import { promptCacheHint } from './prompt-cache'; +import type { AiMessage, GenerateRequest, GenerateResult } from './provider'; import { isTruncated } from './provider'; import { assertNoSecrets } from './redaction'; import type { Respond } from './respond'; @@ -80,76 +78,7 @@ const DEFAULT_TOOL_RESULT_CHARS = 4_000; /** Named in every bound refusal, so the fix names the key the declaration carries. */ const SUBJECT = 'agent'; -export interface AgentBudget extends LlmBudget { - /** - * Token ceiling for the WHOLE run, every turn counted. The one ceiling `llm()` does not need: - * a single call is bounded by `maxTokens`, a loop is bounded by nothing until this is set. - */ - readonly tokensPerRun?: number; -} - -export interface AgentVarsArgs { - readonly input: InferOutput; - readonly ctx: Ctx; -} - -export interface AgentDef< - TInput extends StandardSchemaV1, - TOutput extends StandardSchemaV1, - V extends PromptVars, -> { - readonly model?: ModelId; - readonly input: TInput; - readonly output: TOutput; - readonly prompt: Prompt; - /** Same contract as `llm()`: the one declared place a run loads data. */ - vars(args: AgentVarsArgs): V | Promise; - /** - * The actions the model may call. Each must be `mcp: { expose: true }` — the same predicate an - * external MCP client is filtered by, so an in-app agent and an external one are offered - * exactly the same tools. Listing one that is not exposed is refused at declaration rather than - * dropped, because a tool that reads as offered and silently is not is the worst of both. - */ - readonly tools: readonly AgentTool[]; - /** Hard ceiling on model turns. Reaching it is `X_AGENT_MAX_TURNS`, never a partial answer. */ - readonly maxTurns?: number; - readonly maxToolResultChars?: number; - readonly budget?: AgentBudget; - readonly policy: ActionPolicy; - readonly mcp?: ActionMcp; - /** Enforced completion ceiling PER TURN. The model never sees it. */ - readonly maxTokens?: number; - /** - * Called once per completed model turn, before the answer or the tool calls are acted on. The - * one thing a multi-turn run could not do until 2026-08: a 90-second loop emitted nothing until - * it returned, so a progress indicator, a per-turn spend line and a transcript log all had to be - * hand-rolled outside the framework — which is exactly the loop `agent()` exists to replace. - * - * Observation only: it cannot steer the loop, cannot see the transcript and cannot reach the - * actor. A throw from it FAILS the run rather than being swallowed — it is the app's code on the - * run's own path, and an observer that silently stopped working would be indistinguishable from - * one that is fine. The same facts land on the span as an `agent.turn` event, so a run that - * declares no hook is still readable in a trace. - * - * NOT `.stream()`. Tokens on a screen is a different contract, and `llm()`'s streamed half is - * one turn deep by construction; this is per TURN. - */ - onTurn?(event: AgentTurn): void | Promise; -} - -/** One completed model turn, as an observer sees it. Facts only — no transcript, no actor. */ -export interface AgentTurn { - /** 1-based, so `turn === maxTurns` is the last one this run will take. */ - readonly turn: number; - readonly maxTurns: number; - readonly model: ModelId; - /** Tools this turn asked for, in the order the model emitted them. `respond` is not one. */ - readonly toolCalls: readonly string[]; - readonly stopReason: StopReason; - readonly usage: TokenUsage; - /** This turn's cost alone, integer minor units — never the run's running total. */ - readonly cost: Money; -} +export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent-def'; export function agent< TInput extends StandardSchemaV1, @@ -183,6 +112,21 @@ export function agent< 1, ); finiteCount(SUBJECT, 'maxTokens', def.maxTokens ?? DEFAULT_MAX_TOKENS, 1); + const repeats = finiteCount( + SUBJECT, + 'maxRepeatedToolCalls', + def.maxRepeatedToolCalls ?? DEFAULT_MAX_REPEATED_TOOL_CALLS, + 1, + ); + if (repeats < 2) { + // One "repeat" is the first tool turn of every run: the agent could never use a tool. + throw new AiRequestInvalidError({ + detail: `agent maxRepeatedToolCalls is ${repeats}, which would stop every run at its first tool turn`, + fix: 'maxRepeatedToolCalls: 3 # on the agent — at least 2; it counts consecutive identical turns, the first included', + }); + } + // `AbortSignal.timeout(NaN)` throws a RangeError mid-run; `0` aborts before turn one. + if (def.deadlineMs !== undefined) finiteCount(SUBJECT, 'deadlineMs', def.deadlineMs, 1); // Projected on FIRST RUN, memoised, for the reason above: `agent()` is evaluated at module // scope and a tool's name is stamped by `registerAction` at boot, so naming it here would make // the ordinary `export const publishPost = action(...)` beside it `X_ACTION_UNREGISTERED`. @@ -267,12 +211,19 @@ async function run< const limits = limitsOf(def); // The declaration, its prompt, then the gateway's `defaultModel` — the app's three places. const model = modelForCall('agent', name, def.model, prompt.model); - const vars = await def.vars({ input: args.input, ctx: args.ctx }); + // Armed before anything is loaded: `vars()` is part of the run a deadline bounds. + const deadline = runDeadline(name, def.deadlineMs, args.ctx.signal); + const vars = await deadline.guard(Promise.resolve(def.vars(args))); assertNoSecrets(name, vars); const redact = aiRedactor(); const rawPrompt = prompt.render(vars); const rendered = redact(rawPrompt); - const system = prompt.system === undefined ? undefined : redact(prompt.system); + const extra = + def.system === undefined ? {} : await deadline.guard(Promise.resolve(def.system(args))); + const rawSystem = systemOf(prompt.system, extra, (text) => text); + const system = systemOf(prompt.system, extra, redact); + const promptCache = promptCacheHint(def.promptCache, 'auto', args); + const repeated = toolLoopGuard(name, def.maxRepeatedToolCalls ?? DEFAULT_MAX_REPEATED_TOOL_CALLS); const maxTurns = def.maxTurns ?? DEFAULT_MAX_TURNS; const chars = def.maxToolResultChars ?? DEFAULT_TOOL_RESULT_CHARS; @@ -283,7 +234,8 @@ async function run< 'agent.prompt.hash': prompt.hash, 'agent.tools': adapted.offered.length, 'agent.max_turns': maxTurns, - 'llm.redacted': rendered !== rawPrompt || system !== prompt.system, + 'llm.redacted': rendered !== rawPrompt || !sameSystem(system, rawSystem), + 'llm.prompt_cache': promptCache !== undefined, }); // Rooted in the GATEWAY's own ceilings when no scope is open — an empty root ignored them. @@ -295,6 +247,7 @@ async function run< const base: GenerateRequest = { model, ...(system === undefined ? {} : { system }), + ...(promptCache === undefined ? {} : { promptCache }), messages: [], maxTokens: def.maxTokens ?? DEFAULT_MAX_TOKENS, ...(prompt.effort === undefined ? {} : { effort: prompt.effort }), @@ -302,7 +255,8 @@ async function run< tools: [...adapted.offered, respond.tool], // The caller's own signal, forwarded to the transport. A disconnect has to reach the socket, // not just the top of the next turn: a provider call already in flight is the expensive one. - signal: args.ctx.signal, + // The run's deadline is folded into the same signal, so it reaches the socket too. + signal: deadline.signal, }; return withBudget(ledger, async () => { @@ -313,13 +267,23 @@ async function run< let messages: readonly AiMessage[] = [{ role: 'user', content: rendered }]; let calls = 0; let issues: string | undefined; + let total: AgentRunUsage | undefined; for (let turn = 1; turn <= maxTurns; turn += 1) { // The caller is gone, so unwind instead of finishing. Every turn re-sends the WHOLE // transcript, so a loop that keeps going after a disconnect spends the rest of the run's // budget, runs every remaining tool's side effects, and discards the answer. throwIfAborted(args.ctx); - const result = await gateway.generate({ ...base, messages }); + deadline.check(); + const result = await deadline + .guard(gateway.generate({ ...base, messages })) + .catch((error: unknown) => { + // The caller left mid-call: the aborted socket reads as a provider failure, which is + // RETRYABLE. Say what happened instead. + throwIfAborted(args.ctx); + throw error; + }); + total = addUsage(total, result); const requested = result.toolCalls.filter((call) => call.name !== RESPOND); // The same turn's `respond` calls, kept rather than forgotten: they are never RUN, but // `assistantTurn` replays them, and every replayed `tool_use` has to be answered. @@ -337,6 +301,7 @@ async function run< stopReason: result.stopReason, usage: result.usage, cost: result.cost, + run: total, }); assertAnswerable(result, name, base); @@ -352,10 +317,14 @@ async function run< // result carries the `tool_use` id `runLlmToolCall` was handed, so a fast tool answering // first cannot be paired with a slow tool's call. throwIfAborted(args.ctx); - const results: readonly LlmToolResult[] = await Promise.all( - requested.map((call) => runLlmToolCall(adapted.tools, call, actor)), + deadline.check(); + const results: readonly LlmToolResult[] = await deadline.guard( + Promise.all(requested.map((call) => runLlmToolCall(adapted.tools, call, actor))), ); calls += requested.length; + // Before the next turn is bought: the same calls with the same results cannot lead + // anywhere a further turn would not also lead. + repeated(requested, results); messages = [ ...messages, assistantTurn(result), @@ -408,6 +377,10 @@ async function observeTurn< 'agent.turn.tool_calls': event.toolCalls.length, 'llm.stop': event.stopReason, 'llm.cost.minor': event.cost.minor, + // What the provider's prompt cache did for this turn — the two numbers that say whether the + // stable prefix is really stable. + 'llm.cache.read_tokens': event.usage.cacheReadTokens, + 'llm.cache.write_tokens': event.usage.cacheWriteTokens, }); await def.onTurn?.(event); } diff --git a/packages/ai/src/errors.ts b/packages/ai/src/errors.ts index f5bb4a612..2002dcf64 100644 --- a/packages/ai/src/errors.ts +++ b/packages/ai/src/errors.ts @@ -30,6 +30,8 @@ export const AI_ERROR_CODES = [ 'X_VECTOR_UNSCOPED', 'X_AI_CONTENT_UNSUPPORTED', 'X_AI_MODEL_UNRESOLVED', + 'X_AGENT_TOOL_LOOP', + 'X_AGENT_DEADLINE', ] as const; export type AiErrorCode = (typeof AI_ERROR_CODES)[number]; @@ -61,6 +63,8 @@ export const AI_ERROR_TITLES: Readonly> = { X_VECTOR_UNSCOPED: 'a vector store was read with no tenant bound inside an org request', X_AI_CONTENT_UNSUPPORTED: 'an image or document block the role, model or wire format cannot take', X_AI_MODEL_UNRESOLVED: 'a model call named no model', + X_AGENT_TOOL_LOOP: 'an agent repeated one tool call with one result and was stopped', + X_AGENT_DEADLINE: 'an agent ran past its declared deadline and was stopped', }; // Titles must be registered for `format()` to render the contract's first line. Unconditional and diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 6a48e3b53..289fd9e3a 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -4,12 +4,15 @@ /** Re-exported so an `llm` file needs one import, not two. Same object as schema's. */ export type { Infer } from '@ultimat3/schema'; export { t } from '@ultimat3/schema'; -export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent'; export { agent } from './agent'; +export type { AgentBudget, AgentDef, AgentTurn, AgentVarsArgs } from './agent-def'; +export { AgentDeadlineError, AgentToolLoopError } from './agent-errors'; export type { AgentBudgetFact, AgentFact } from './agent-facts'; export { describeAgents, resetAgents } from './agent-facts'; export type { AgentJobOptions } from './agent-job'; export { agentJob } from './agent-job'; +export type { AgentSystem } from './agent-system'; +export type { AgentRunUsage } from './agent-usage'; export type { BudgetKeys, BudgetLedger, @@ -177,6 +180,15 @@ export { promptVersions, resetPrompts, } from './prompt'; +export type { + CacheWire, + PromptCacheHint, + PromptCacheOption, + PromptCacheRule, + SystemParts, + SystemPrompt, +} from './prompt-cache'; +export { PROMPT_CACHE_RULES, promptCacheRule, SESSION_KEY_MAX, systemText } from './prompt-cache'; export { promptFences } from './prompt-fence'; export type { AiContentBlock, diff --git a/packages/ai/src/llm-prompt-cache.test.ts b/packages/ai/src/llm-prompt-cache.test.ts new file mode 100644 index 000000000..6a938b17a --- /dev/null +++ b/packages/ai/src/llm-prompt-cache.test.ts @@ -0,0 +1,57 @@ +/** + * `llm()` and provider prompt caching: OFF unless the declaration asks. A one-shot call that marks + * its prefix pays the write premium for a cache nothing may read. + */ + +import { describe, expect, test } from 'bun:test'; +import { ANSWER, ctxFor, declare, install, POST_ID, promptFor, stub } from './llm-fixture'; +import { useFixtureModels } from './model-fixture'; +import { registerModel } from './models'; +import { chatCompletionBody } from './openai-body'; +import { resetAiRuntime } from './runtime'; + +useFixtureModels(); + +const run = async (extra: Parameters[1]) => { + resetAiRuntime(); + const { provider, seen } = stub(ANSWER); + install(provider); + await declare(promptFor(), extra)({ postId: POST_ID }, { ctx: ctxFor('u-1', 'org-1') }); + return seen[0] ?? expect.unreachable('the model was not called'); +}; + +describe('llm() sends no cache mark by default', () => { + test('no option: no hint on the request, and no mark on an Anthropic-on-OpenRouter body', async () => { + const request = await run({}); + expect(request.promptCache).toBeUndefined(); + const model = 'anthropic/claude-sonnet-5.5'; + registerModel({ + id: model, + contextWindow: 1_000_000, + maxOutput: 128_000, + inputPerMillion: { minor: 200, currency: 'USD' }, + outputPerMillion: { minor: 1_000, currency: 'USD' }, + cacheMinimumTokens: 512, + reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, + }); + const body = chatCompletionBody({ + request: { ...request, model, system: 'a long stable system prompt' }, + model, + provider: 'openrouter', + stream: false, + baseUrl: 'https://openrouter.ai/api/v1', + }); + expect(JSON.stringify(body)).not.toContain('cache_control'); + expect(body['prompt_cache_key']).toBeUndefined(); + }); + + test("'auto' and a session key are the declaration's to ask for", async () => { + expect((await run({ promptCache: 'auto' })).promptCache).toEqual({}); + expect((await run({ promptCache: 'off' })).promptCache).toBeUndefined(); + const keyed = await run({ promptCache: { session: ({ ctx }) => `sum:${ctx.locale}` } }); + expect(keyed.promptCache?.session).toStartWith('sum:'); + expect((await run({ promptCache: { session: 'summaries' } })).promptCache).toEqual({ + session: 'summaries', + }); + }); +}); diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 044f0b07e..73b19b01d 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -45,6 +45,8 @@ import { modelForCall } from './model-resolve'; import type { ModelId } from './models'; import { moreCapableThan } from './models'; import type { Prompt, PromptVars } from './prompt'; +import type { PromptCacheOption } from './prompt-cache'; +import { promptCacheHint } from './prompt-cache'; import type { AiMessage, GenerateRequest, GenerateResult } from './provider'; import { isTruncated } from './provider'; import { assertNoSecrets } from './redaction'; @@ -94,6 +96,13 @@ export interface LlmDef< */ vars(args: LlmVarsArgs): V | Promise; readonly cache?: LlmCache>; + /** + * PROVIDER prompt caching (`./prompt-cache`) — not `cache`, which is this package's own semantic + * cache. Default `'off'`: a one-shot call marked for caching pays the write premium (1.25x on + * Anthropic) for a prefix nothing may ever read back. `'auto'` is for a call made many times + * inside the cache's lifetime with one long `system`; the repair turn reads it back either way. + */ + readonly promptCache?: PromptCacheOption>; readonly budget?: LlmBudget; readonly policy: ActionPolicy; readonly mcp?: ActionMcp; @@ -207,6 +216,7 @@ async function generate< const rendered = redact(rawPrompt); const system = prompt.system === undefined ? undefined : redact(prompt.system); const redacted = rendered !== rawPrompt || system !== prompt.system; + const promptCache = promptCacheHint(def.promptCache, 'off', args); return withSpan('ai.llm', async (span) => { span.setAttributes({ @@ -239,6 +249,7 @@ async function generate< const base: GenerateRequest = { model, ...(system === undefined ? {} : { system }), + ...(promptCache === undefined ? {} : { promptCache }), messages: [{ role: 'user', content: rendered }], maxTokens: def.maxTokens ?? DEFAULT_MAX_TOKENS, ...(prompt.effort === undefined ? {} : { effort: prompt.effort }), diff --git a/packages/ai/src/openai-body.ts b/packages/ai/src/openai-body.ts index 0b800df52..bdbc8d19d 100644 --- a/packages/ai/src/openai-body.ts +++ b/packages/ai/src/openai-body.ts @@ -9,6 +9,7 @@ import { AiRequestInvalidError } from './errors'; import type { Effort, ModelId, ThinkingMode } from './models'; import { modelSpec } from './models'; import { toOpenAiMessages, toOpenAiTools, toolChoiceFor } from './openai-messages'; +import { promptCacheRule, SESSION_KEY_MAX } from './prompt-cache'; import type { GenerateRequest } from './provider'; export interface ChatCompletionBodyInput { @@ -18,6 +19,8 @@ export interface ChatCompletionBodyInput { /** The provider's `name`, so a content refusal names the endpoint that cannot take the block. */ readonly provider: string; readonly stream: boolean; + /** The endpoint, for `PROMPT_CACHE_RULES` — which host this is decides what a cache hint sends. */ + readonly baseUrl?: string | undefined; } /** @@ -33,9 +36,12 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record = { model, - messages: toOpenAiMessages(request.system, request.messages, { provider, model }), + messages: toOpenAiMessages(request.system, request.messages, { provider, model }, marked), max_completion_tokens: Math.min(request.maxTokens, spec.maxOutput), ...reasoningFields(model, request.effort, request.thinking), }; @@ -47,6 +53,11 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record 0) { body['stop'] = request.stopSequences; } + // OpenAI's own grouping key. OpenRouter's session rides a header instead (`sessionHeaders`), so + // the body never carries two spellings of one key. + if (hint?.session !== undefined && rule.session === 'prompt_cache_key') { + body['prompt_cache_key'] = sessionKey(hint.session); + } if (stream) { body['stream'] = true; // Without this the final chunk carries no `usage` and the budget reconciles against nothing — @@ -57,6 +68,32 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record> { + const session = request.promptCache?.session; + if (session === undefined) return {}; + if (promptCacheRule('openai', baseUrl, model).session !== 'x-session-id') return {}; + return { 'x-session-id': sessionKey(session) }; +} + +/** + * Refused, never cut: a truncated key is a DIFFERENT key that two tenants can share, and a control + * character in a header value is a request the runtime refuses with no code at all. + */ +function sessionKey(session: string): string { + if (session !== '' && session.length <= SESSION_KEY_MAX && !/[^\x20-\x7e]/.test(session)) { + return session; + } + throw new AiRequestInvalidError({ + detail: `the prompt-cache session key is ${session.length} characters; it must be 1 to ${SESSION_KEY_MAX} printable ASCII characters to travel as a header`, + fix: "promptCache: { session: 'kb-agent' } # a short, stable, non-secret name for the runs that share a prefix", + }); +} + /** * The reasoning half, shaped for one model. Refused LOCALLY when the model's spec says the endpoint * has no such control, for the reason models.ts states: a round trip to learn a rule the registry diff --git a/packages/ai/src/openai-messages.ts b/packages/ai/src/openai-messages.ts index 6cef6074a..d84dd3c7e 100644 --- a/packages/ai/src/openai-messages.ts +++ b/packages/ai/src/openai-messages.ts @@ -8,6 +8,8 @@ import type { AiMediaBlock } from './content-blocks'; import { isMediaBlock } from './content-blocks'; import { AiContentUnsupportedError } from './content-errors'; +import type { SystemPrompt } from './prompt-cache'; +import { CACHE_MARK, systemParts, systemText } from './prompt-cache'; import type { AiContentBlock, AiMessage } from './provider'; import type { JsonSchema, LlmTool } from './tools'; @@ -20,7 +22,12 @@ export interface OpenAiToolCall { /** A user message's parts, when it carries more than prose. Field names are the format's own. */ export type OpenAiContentPart = - | { readonly type: 'text'; readonly text: string } + | { + readonly type: 'text'; + readonly text: string; + /** A prompt-cache breakpoint — sent ONLY where `PROMPT_CACHE_RULES` says `parts`. */ + readonly cache_control?: typeof CACHE_MARK; + } | { readonly type: 'image_url'; readonly image_url: { readonly url: string } } | { readonly type: 'file'; @@ -28,14 +35,18 @@ export type OpenAiContentPart = }; export type OpenAiMessage = - | { readonly role: 'system'; readonly content: string } + | { readonly role: 'system'; readonly content: string | readonly OpenAiContentPart[] } | { readonly role: 'user'; readonly content: string | readonly OpenAiContentPart[] } | { readonly role: 'assistant'; readonly content?: string | undefined; readonly tool_calls?: readonly OpenAiToolCall[] | undefined; } - | { readonly role: 'tool'; readonly tool_call_id: string; readonly content: string }; + | { + readonly role: 'tool'; + readonly tool_call_id: string; + readonly content: string | readonly OpenAiContentPart[]; + }; export interface OpenAiFunctionTool { readonly type: 'function'; @@ -68,12 +79,17 @@ export interface OpenAiToolChoice { * speaking this format — vLLM, Ollama, LiteLLM, Together — knows only `system`. OpenAI accepts it. */ export function toOpenAiMessages( - system: string | undefined, + system: SystemPrompt | undefined, messages: readonly AiMessage[], target: MediaTarget, + /** Whether this endpoint and model take explicit `cache_control` parts (`./prompt-cache`). */ + marked = false, ): readonly OpenAiMessage[] { const out: OpenAiMessage[] = []; - if (system !== undefined && system !== '') out.push({ role: 'system', content: system }); + // One string, stable half first, wherever nothing is marked: every server in the family takes + // it, and an implicit cache is a prefix match. + const text = systemText(system); + if (text !== '') out.push({ role: 'system', content: marked ? systemPartsOf(system) : text }); messages.forEach((message, m) => { if (typeof message.content === 'string') { out.push({ role: message.role, content: message.content }); @@ -84,9 +100,38 @@ export function toOpenAiMessages( out.push(...userTurn(message.content, m, target)); } }); + if (marked) markTail(out); return out; } +/** The system message as parts: the stable one marked, the per-run one after it and unmarked. */ +function systemPartsOf(system: SystemPrompt | undefined): readonly OpenAiContentPart[] { + const { stable, run } = systemParts(system); + const parts: OpenAiContentPart[] = []; + if (stable !== '') parts.push({ type: 'text', text: stable, cache_control: CACHE_MARK }); + if (run !== '') parts.push({ type: 'text', text: run }); + return parts; +} + +/** + * Mark the LAST message's last part — the breakpoint the next turn reads back. A `user` or `tool` + * message only: those are the roles whose content this wire takes as parts, and a transcript sent + * to a model always ends on one of them. + */ +function markTail(out: OpenAiMessage[]): void { + const tail = out.at(-1); + if (tail === undefined || (tail.role !== 'user' && tail.role !== 'tool')) return; + const parts: readonly OpenAiContentPart[] = + typeof tail.content === 'string' ? [{ type: 'text', text: tail.content }] : tail.content; + const last = parts.at(-1); + // Only a text part carries the mark here, and never an empty one. + if (last === undefined || last.type !== 'text' || last.text === '') return; + out[out.length - 1] = { + ...tail, + content: [...parts.slice(0, -1), { ...last, cache_control: CACHE_MARK }], + }; +} + /** Who a media refusal names: this provider, and the model the request is for. */ export interface MediaTarget { readonly provider: string; diff --git a/packages/ai/src/openai-provider.ts b/packages/ai/src/openai-provider.ts index 06415340b..736653ca5 100644 --- a/packages/ai/src/openai-provider.ts +++ b/packages/ai/src/openai-provider.ts @@ -14,7 +14,7 @@ import { AiKeyMissingError, AiRequestInvalidError, AiTransportError } from './er import type { AiFetch } from './fetch-seam'; import { resolveModel } from './model-resolve'; import type { ModelId } from './models'; -import { chatCompletionBody } from './openai-body'; +import { chatCompletionBody, sessionHeaders } from './openai-body'; import type { ChatAnswer } from './openai-wire'; import { ChatCompletionStream, parseChatCompletion } from './openai-wire'; import type { @@ -115,10 +115,12 @@ class OpenAiProvider implements Provider { // this provider's first listed model — the app's three places are the only ones. const model = resolveModel('provider', request.model); if (requiresStreaming({ ...request, model })) return this.assemble(request); + const { baseUrl } = this.config; const response = await this.send( - chatCompletionBody({ request, model, stream: false, provider: this.name }), + chatCompletionBody({ request, model, stream: false, provider: this.name, baseUrl }), false, request.signal, + sessionHeaders(request, model, baseUrl), ); const answer = parseChatCompletion((await response.json()) as unknown, this.name); return this.result(request, model, answer); @@ -126,10 +128,12 @@ class OpenAiProvider implements Provider { async *stream(request: GenerateRequest): AsyncIterable { const model = resolveModel('provider', request.model); + const { baseUrl } = this.config; const response = await this.send( - chatCompletionBody({ request, model, stream: true, provider: this.name }), + chatCompletionBody({ request, model, stream: true, provider: this.name, baseUrl }), true, request.signal, + sessionHeaders(request, model, baseUrl), ); if (response.body === null) { throw new AiTransportError({ @@ -187,6 +191,7 @@ class OpenAiProvider implements Provider { body: Record, streaming: boolean, signal: AbortSignal | undefined, + extra: Readonly>, ): Promise { const apiKey = this.apiKey(); const doFetch: AiFetch = this.config.fetch ?? fetch; @@ -198,6 +203,8 @@ class OpenAiProvider implements Provider { : { authorization: `Bearer ${apiKey}` }), 'content-type': 'application/json', accept: streaming ? 'text/event-stream' : 'application/json', + // The prompt-cache session, when the request carries one and this host routes by it. + ...extra, // Last, so a caller can replace the credential header outright for a gateway that wants // its own scheme. Nothing here is ever logged. ...this.config.headers, diff --git a/packages/ai/src/openai-wire.test.ts b/packages/ai/src/openai-wire.test.ts index 2f4471d3e..37ad350cf 100644 --- a/packages/ai/src/openai-wire.test.ts +++ b/packages/ai/src/openai-wire.test.ts @@ -175,6 +175,45 @@ describe('usage', () => { }); }); + test('reads a cache WRITE out of the prompt count too, and survives every missing field', () => { + // OpenRouter, turn one of a marked run: 6,000 tokens written, nothing read yet. + expect( + parseOpenAiUsage({ + prompt_tokens: 6_200, + completion_tokens: 40, + prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: 6_000 }, + cost: 0.0158, + }), + ).toEqual({ inputTokens: 200, outputTokens: 40, cacheReadTokens: 0, cacheWriteTokens: 6_000 }); + // Turn two: the prefix read, the new tail written. + expect( + parseOpenAiUsage({ + prompt_tokens: 6_500, + completion_tokens: 40, + prompt_tokens_details: { cached_tokens: 6_000, cache_write_tokens: 450 }, + }), + ).toEqual({ inputTokens: 50, outputTokens: 40, cacheReadTokens: 6_000, cacheWriteTokens: 450 }); + // No details object, a null one, a details object with neither counter, non-numbers. + const bare = { inputTokens: 100, outputTokens: 5, cacheReadTokens: 0, cacheWriteTokens: 0 }; + expect(parseOpenAiUsage({ prompt_tokens: 100, completion_tokens: 5 })).toEqual(bare); + for (const details of [null, {}, 'x', { cached_tokens: null, cache_write_tokens: '9' }]) { + expect( + parseOpenAiUsage({ + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: details, + }), + ).toEqual(bare); + } + // Counters that exceed the prompt they are part of never bill a token that was not sent. + expect( + parseOpenAiUsage({ + prompt_tokens: 100, + prompt_tokens_details: { cached_tokens: 80, cache_write_tokens: 500 }, + }), + ).toEqual({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 80, cacheWriteTokens: 20 }); + }); + test('is undefined — never zero — when the endpoint reported none', () => { // Zero would read as a free call and refund the whole reservation. The provider substitutes an // estimate; it can only do that if this half says "nothing arrived" rather than "nothing spent". diff --git a/packages/ai/src/openai-wire.ts b/packages/ai/src/openai-wire.ts index 25d5ff3c6..dd229d5e9 100644 --- a/packages/ai/src/openai-wire.ts +++ b/packages/ai/src/openai-wire.ts @@ -75,9 +75,14 @@ export function parseFinishReason(raw: unknown): StopReason | undefined { /** * `usage`, or `undefined` when the payload carried none. * - * `prompt_tokens` INCLUDES the cached prefix on this wire, where Anthropic's `input_tokens` - * excludes it — so the cached half is subtracted out before it is reported as `cacheReadTokens`. - * Left in place it would be billed twice: once at the full input rate and again at the cache rate. + * `prompt_tokens` is the WHOLE prompt on this wire — the part read from a cache and the part + * written to one included — where Anthropic's `input_tokens` excludes both. So both are subtracted + * out before they are reported under their own counters. Left in place a cached token would be + * billed twice: once at the full input rate and again at the cache rate. + * + * `cache_write_tokens` is OpenRouter's (`prompt_tokens_details.cache_write_tokens`, "Number of + * tokens written to the cache"), reported for a model whose vendor bills a write. An endpoint that + * never sends it — OpenAI's own, where a write is free — reads as zero. */ export function parseOpenAiUsage(raw: unknown): TokenUsage | undefined { const record = asRecord(raw); @@ -85,14 +90,18 @@ export function parseOpenAiUsage(raw: unknown): TokenUsage | undefined { const prompt = countOf(record['prompt_tokens']); const completion = countOf(record['completion_tokens']); if (prompt === undefined && completion === undefined) return undefined; - const cached = countOf(asRecord(record['prompt_tokens_details'])?.['cached_tokens']) ?? 0; + const details = asRecord(record['prompt_tokens_details']); + const total = prompt ?? 0; + // Each clamped to what is left of the prompt: a counter larger than the prompt it is part of is + // the endpoint's mistake, and an unclamped one bills tokens that were never sent. + const cached = Math.min(countOf(details?.['cached_tokens']) ?? 0, total); + const written = Math.min(countOf(details?.['cache_write_tokens']) ?? 0, total - cached); return { - inputTokens: Math.max((prompt ?? 0) - cached, 0), + inputTokens: total - cached - written, // `completion_tokens` already contains `reasoning_tokens`; adding them is a double count. outputTokens: completion ?? 0, cacheReadTokens: cached, - // Caching is automatic here and carries no write surcharge, so there is nothing to report. - cacheWriteTokens: 0, + cacheWriteTokens: written, }; } diff --git a/packages/ai/src/prompt-cache-blocks.ts b/packages/ai/src/prompt-cache-blocks.ts new file mode 100644 index 000000000..88c66fd1d --- /dev/null +++ b/packages/ai/src/prompt-cache-blocks.ts @@ -0,0 +1,61 @@ +// Single responsibility: the Anthropic Messages wire's half of prompt caching — `system` as text +// blocks, and `cache_control` on the blocks ./prompt-cache's `blocks` rule names. Pure, so the body +// a test asserts is the body the socket gets. + +import type { SystemPrompt } from './prompt-cache'; +import { CACHE_MARK, systemParts } from './prompt-cache'; +import type { AiMessage } from './provider'; + +type Marked = T & { readonly cache_control?: typeof CACHE_MARK }; +type TextBlock = Marked<{ readonly type: 'text'; readonly text: string }>; + +/** How many trailing messages carry a mark. With the system block: three of the four breakpoints. */ +const MARKED_TAIL = 2; + +/** + * `system`, as the wire takes it. A plain string with no mark stays the string every existing + * request sent. Otherwise blocks, the stable one FIRST and the only one marked: it is the half + * other runs share, and the per-run half after it is covered by the transcript's own marks. + * An empty part is dropped — an empty text block is a 400, and a mark on one is too. + */ +export function anthropicSystem( + system: SystemPrompt | undefined, + marked: boolean, +): string | readonly TextBlock[] | undefined { + if (system === undefined) return undefined; + if (typeof system === 'string' && !marked) return system; + const { stable, run } = systemParts(system); + const blocks: TextBlock[] = []; + if (stable !== '') { + blocks.push({ type: 'text', text: stable, ...(marked ? { cache_control: CACHE_MARK } : {}) }); + } + if (run !== '') blocks.push({ type: 'text', text: run }); + return blocks.length === 0 ? undefined : blocks; +} + +/** + * The transcript, with the last block of the last two messages marked. The tail is what the NEXT + * request reads back; the one before it gives the lookup a second start, so a turn that appended + * more than the 20 blocks one breakpoint looks back over still finds the previous turn's entry. + */ +export function anthropicMessages( + messages: readonly AiMessage[], + marked: boolean, +): readonly { readonly role: AiMessage['role']; readonly content: unknown }[] { + const from = marked ? messages.length - MARKED_TAIL : messages.length; + return messages.map((message, index) => ({ + role: message.role, + content: index >= from ? markedContent(message.content) : message.content, + })); +} + +function markedContent(content: AiMessage['content']): unknown { + if (typeof content === 'string') { + // An empty text block cannot be marked (or sent); the string goes as it stood. + return content === '' ? content : [{ type: 'text', text: content, cache_control: CACHE_MARK }]; + } + const last = content.length - 1; + return content.map((block, index) => + index === last ? { ...block, cache_control: CACHE_MARK } : block, + ); +} diff --git a/packages/ai/src/prompt-cache.test.ts b/packages/ai/src/prompt-cache.test.ts new file mode 100644 index 000000000..0fac7e031 --- /dev/null +++ b/packages/ai/src/prompt-cache.test.ts @@ -0,0 +1,391 @@ +/** + * What LEAVES the process when a request carries a prompt-cache hint — one golden request per rule + * in `PROMPT_CACHE_RULES`, asserted on the body and headers the socket is handed. + */ + +import { describe, expect, test } from 'bun:test'; +import { registerModel } from './models'; +import { openAiProvider } from './openai-provider'; +import { promptCacheRule, SESSION_KEY_MAX, systemText } from './prompt-cache'; +import type { GenerateRequest } from './provider'; +import { anthropicProvider, estimateCost } from './provider'; + +const usd = (minor: number) => ({ minor, currency: 'USD' }); +const ROW = { + contextWindow: 200_000, + maxOutput: 8_000, + inputPerMillion: usd(300), + outputPerMillion: usd(1_500), + cacheMinimumTokens: 1_024, + reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, +} as const; +const CLAUDE = 'anthropic/claude-sonnet-5.5'; +const GEMINI = 'google/gemini-2.5-flash'; +const GPT = 'gpt-5.6-luna'; +const NATIVE = 'claude-sonnet-5-5'; + +function register(): void { + for (const id of [CLAUDE, GEMINI, GPT, NATIVE]) registerModel({ id, ...ROW }); +} + +const MARK = { type: 'ephemeral' }; +const STABLE = 'You are the firm’s assistant. RULES…'; +const RUN = 'Today is 2026-10-11.'; + +/** A three-message transcript: the shape turn 2 of a tool loop sends. */ +const transcript: GenerateRequest['messages'] = [ + { role: 'user', content: 'Resolve order 7.' }, + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'call-1', name: 'lookup', input: { id: '7' } }], + }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call-1', content: 'shipped' }] }, +]; + +interface Sent { + readonly url: string; + readonly headers: Record; + readonly body: Record; +} + +function capture(reply: unknown): { sent: Sent[]; fetch: typeof fetch } { + const sent: Sent[] = []; + const fake = (async (url: string | URL | Request, init?: RequestInit) => { + sent.push({ + url: String(url), + headers: { ...(init?.headers as Record) }, + body: JSON.parse(String(init?.body)) as Record, + }); + return new Response(JSON.stringify(reply), { status: 200 }); + }) as typeof fetch; + return { sent, fetch: fake }; +} + +const CHAT_REPLY = { + choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 1 }, +}; + +async function openRouter(request: GenerateRequest, baseUrl = 'https://openrouter.ai/api/v1') { + register(); + const { sent, fetch } = capture(CHAT_REPLY); + const provider = openAiProvider({ apiKey: 'k', baseUrl, models: [CLAUDE, GEMINI, GPT], fetch }); + await provider.generate(request); + const only = sent[0]; + if (only === undefined) return expect.unreachable('no request was sent'); + return only; +} + +/** Every `cache_control` anywhere in a body — the count a vendor caps at four. */ +function marks(value: unknown): number { + if (Array.isArray(value)) return value.reduce((sum: number, item) => sum + marks(item), 0); + if (typeof value !== 'object' || value === null) return 0; + return Object.entries(value).reduce( + (sum, [key, child]) => sum + (key === 'cache_control' ? 1 : marks(child)), + 0, + ); +} + +describe('the rule table', () => { + test('first match wins, per wire, host and model prefix', () => { + const rule = (wire: 'anthropic' | 'openai', url: string | undefined, model: string) => { + const { marks, session } = promptCacheRule(wire, url, model); + return [marks, session]; + }; + expect(rule('anthropic', undefined, NATIVE)).toEqual(['blocks', 'none']); + expect(rule('openai', 'https://openrouter.ai/api/v1', CLAUDE)).toEqual([ + 'parts', + 'x-session-id', + ]); + expect(rule('openai', 'https://openrouter.ai/api/v1', GEMINI)).toEqual([ + 'none', + 'x-session-id', + ]); + expect(rule('openai', undefined, GPT)).toEqual(['none', 'prompt_cache_key']); + expect(rule('openai', 'https://api.openai.com/v1', GPT)).toEqual(['none', 'prompt_cache_key']); + // An endpoint the table has never heard of gets nothing it might reject. + expect(rule('openai', 'http://vllm:8000/v1', 'acme-70b')).toEqual(['none', 'none']); + // A look-alike host is not the host. + expect(rule('openai', 'https://openrouter.ai.evil.test/v1', CLAUDE)).toEqual(['none', 'none']); + expect(rule('openai', 'not a url', CLAUDE)).toEqual(['none', 'none']); + }); + + test('the two system parts read as one text, stable first', () => { + expect(systemText(undefined)).toBe(''); + expect(systemText('s')).toBe('s'); + expect(systemText({ stable: 'a', run: 'b' })).toBe('a\n\nb'); + expect(systemText({ stable: 'a' })).toBe('a'); + expect(systemText({ stable: '', run: 'b' })).toBe('b'); + }); +}); + +describe('Anthropic model on OpenRouter (chat-completions wire)', () => { + test('the stable system part and the tail carry the mark; never more than four', async () => { + const { body, headers } = await openRouter({ + model: CLAUDE, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 100, + promptCache: {}, + }); + expect(body['messages']).toEqual([ + { + role: 'system', + content: [ + { type: 'text', text: STABLE, cache_control: MARK }, + { type: 'text', text: RUN }, + ], + }, + { role: 'user', content: 'Resolve order 7.' }, + { + role: 'assistant', + tool_calls: [ + { id: 'call-1', type: 'function', function: { name: 'lookup', arguments: '{"id":"7"}' } }, + ], + }, + { + role: 'tool', + tool_call_id: 'call-1', + content: [{ type: 'text', text: 'shipped', cache_control: MARK }], + }, + ]); + expect(marks(body)).toBe(2); + expect(marks(body)).toBeLessThanOrEqual(4); + // No key, no session; and routing is never steered. + expect(headers['x-session-id']).toBeUndefined(); + expect(body['session_id']).toBeUndefined(); + expect(body['provider']).toBeUndefined(); + expect(body['prompt_cache_key']).toBeUndefined(); + expect(body['usage']).toBeUndefined(); + }); + + test('a plain-string system is one marked part; a prose tail is marked as a part', async () => { + const { body } = await openRouter({ + model: CLAUDE, + system: STABLE, + messages: [{ role: 'user', content: 'hello' }], + maxTokens: 100, + promptCache: {}, + }); + expect(body['messages']).toEqual([ + { role: 'system', content: [{ type: 'text', text: STABLE, cache_control: MARK }] }, + { role: 'user', content: [{ type: 'text', text: 'hello', cache_control: MARK }] }, + ]); + }); + + test('a session key rides the x-session-id header, and only with a key', async () => { + const request: GenerateRequest = { + model: CLAUDE, + system: STABLE, + messages: transcript, + maxTokens: 100, + }; + const keyed = await openRouter({ ...request, promptCache: { session: 'kb-agent:org-1' } }); + expect(keyed.headers['x-session-id']).toBe('kb-agent:org-1'); + expect(keyed.body['prompt_cache_key']).toBeUndefined(); + const bare = await openRouter({ ...request, promptCache: {} }); + expect(bare.headers['x-session-id']).toBeUndefined(); + }); + + test('no hint, no mark: the request is byte-for-byte what it was', async () => { + const { body, headers } = await openRouter({ + model: CLAUDE, + system: STABLE, + messages: transcript, + maxTokens: 100, + }); + expect(marks(body)).toBe(0); + expect((body['messages'] as unknown[])[0]).toEqual({ role: 'system', content: STABLE }); + expect(headers['x-session-id']).toBeUndefined(); + }); + + test('a session key over the host limit is refused before the socket', async () => { + register(); + const { sent, fetch } = capture(CHAT_REPLY); + const provider = openAiProvider({ + apiKey: 'k', + baseUrl: 'https://openrouter.ai/api/v1', + models: [CLAUDE], + fetch, + }); + const call = provider.generate({ + model: CLAUDE, + messages: transcript, + maxTokens: 100, + promptCache: { session: 'x'.repeat(SESSION_KEY_MAX + 1) }, + }); + await expect(call).rejects.toMatchObject({ code: 'X_AI_REQUEST_INVALID' }); + expect(sent).toHaveLength(0); + }); +}); + +describe('a non-Anthropic model on OpenRouter', () => { + test('Gemini gets NO explicit mark — it would turn free implicit caching into a paid write', async () => { + const { body, headers } = await openRouter({ + model: GEMINI, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 100, + promptCache: { session: 's-1' }, + }); + expect(marks(body)).toBe(0); + // One string, stable half FIRST: implicit caching is a prefix match. + expect((body['messages'] as unknown[])[0]).toEqual({ + role: 'system', + content: `${STABLE}\n\n${RUN}`, + }); + expect((body['messages'] as unknown[]).at(-1)).toEqual({ + role: 'tool', + tool_call_id: 'call-1', + content: 'shipped', + }); + // Stickiness is the host's, whatever the model: implicit caches are per upstream too. + expect(headers['x-session-id']).toBe('s-1'); + expect(body['provider']).toBeUndefined(); + }); +}); + +describe('OpenAI direct', () => { + test('a session key is prompt_cache_key; nothing is marked', async () => { + const request: GenerateRequest = { + model: GPT, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 100, + }; + const keyed = await openRouter( + { ...request, promptCache: { session: 'kb' } }, + 'https://api.openai.com/v1', + ); + expect(keyed.body['prompt_cache_key']).toBe('kb'); + expect(keyed.headers['x-session-id']).toBeUndefined(); + expect(marks(keyed.body)).toBe(0); + const bare = await openRouter({ ...request, promptCache: {} }, 'https://api.openai.com/v1'); + expect(bare.body['prompt_cache_key']).toBeUndefined(); + }); + + test('an endpoint the table does not name gets neither mark nor session', async () => { + const { body, headers } = await openRouter( + { + model: CLAUDE, + system: STABLE, + messages: transcript, + maxTokens: 100, + promptCache: { session: 's' }, + }, + 'https://llm.acme.internal/v1', + ); + expect(marks(body)).toBe(0); + expect(headers['x-session-id']).toBeUndefined(); + expect(body['prompt_cache_key']).toBeUndefined(); + }); +}); + +describe('Anthropic Messages wire', () => { + const provider = () => { + register(); + return anthropicProvider({ models: [NATIVE], apiKey: 'k' }); + }; + + test('the stable system block and the last block of the last two messages; three marks', () => { + const body = provider().body({ + model: NATIVE, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 100, + promptCache: { session: 'ignored-here' }, + }); + expect(body['system']).toEqual([ + { type: 'text', text: STABLE, cache_control: MARK }, + { type: 'text', text: RUN }, + ]); + expect(body['messages']).toEqual([ + { role: 'user', content: 'Resolve order 7.' }, + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'call-1', + name: 'lookup', + input: { id: '7' }, + cache_control: MARK, + }, + ], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call-1', content: 'shipped', cache_control: MARK }, + ], + }, + ]); + expect(marks(body)).toBe(3); + }); + + test('a one-message request marks it once; a string message becomes one marked block', () => { + const body = provider().body({ + model: NATIVE, + system: STABLE, + messages: [{ role: 'user', content: 'hello' }], + maxTokens: 100, + promptCache: {}, + }); + expect(body['system']).toEqual([{ type: 'text', text: STABLE, cache_control: MARK }]); + expect(body['messages']).toEqual([ + { role: 'user', content: [{ type: 'text', text: 'hello', cache_control: MARK }] }, + ]); + expect(marks(body)).toBe(2); + }); + + test('no hint: a string system stays a string, two parts are two unmarked blocks', () => { + const plain = provider().body({ + model: NATIVE, + system: STABLE, + messages: transcript, + maxTokens: 9, + }); + expect(plain['system']).toBe(STABLE); + expect(marks(plain)).toBe(0); + const parts = provider().body({ + model: NATIVE, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 9, + }); + expect(parts['system']).toEqual([ + { type: 'text', text: STABLE }, + { type: 'text', text: RUN }, + ]); + expect(marks(parts)).toBe(0); + }); + + test('an empty stable part is never sent as an empty block, and never marked', () => { + const body = provider().body({ + model: NATIVE, + system: { stable: '', run: RUN }, + messages: [{ role: 'user', content: 'hello' }], + maxTokens: 100, + promptCache: {}, + }); + expect(body['system']).toEqual([{ type: 'text', text: RUN }]); + expect(marks(body)).toBe(1); + }); +}); + +describe('the pre-flight estimate', () => { + test('a hinted request is priced as a cache WRITE, the dearest the prompt can be billed', () => { + register(); + const text = 'x'.repeat(4_000_000); + const request: GenerateRequest = { + model: NATIVE, + messages: [{ role: 'user', content: text }], + maxTokens: 1, + }; + const plain = estimateCost(request).minor; + const hinted = estimateCost({ ...request, promptCache: {} }).minor; + // 1M tokens at 300 minor, then 1.25x; the single output token rounds up to one minor unit. + expect(plain).toBe(301); + expect(hinted).toBe(376); + }); +}); diff --git a/packages/ai/src/prompt-cache.ts b/packages/ai/src/prompt-cache.ts new file mode 100644 index 000000000..921c5fd41 --- /dev/null +++ b/packages/ai/src/prompt-cache.ts @@ -0,0 +1,173 @@ +// Single responsibility: PROVIDER prompt caching — the neutral request shapes (a system prompt in +// a stable and a per-run part, the hint that asks for the prefix to be cached) and the table that +// says how each wire and host is told. Not the gateway's response cache, not `llm()`'s semantic one. +// +// THE RULE, and where it comes from (both pages read 2026-10-11): +// +// Anthropic — https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching +// "Place `cache_control` directly on individual content blocks"; the shape is +// `"cache_control": { "type": "ephemeral" }` (5-minute TTL, the default). +// "Prompt caching references the entire prompt: `tools`, `system`, and `messages` (in that +// order), up to and including the block designated with `cache_control`." +// "You can define up to 4 cache breakpoints" — a fifth is a 400. +// "The system checks at most 20 positions per breakpoint, counting the breakpoint itself." +// "Shorter prompts cannot be cached, even if marked with `cache_control`. Any requests to cache +// fewer than this number of tokens will be processed without caching, and no error is +// returned." Minimums are per model (512 to 4,096 tokens) — `ModelSpec.cacheMinimumTokens`. +// "5-minute cache write tokens are 1.25 times the base input tokens price … Cache read tokens +// are 0.1 times the base input tokens price" (0.05x and 0.025x on named models). +// +// OpenRouter — https://openrouter.ai/docs/features/prompt-caching +// Anthropic models: `cache_control` on a `text` part inside a message's `content` array; +// "There is a limit of four explicit breakpoints." +// Gemini: "Gemini 2.5 series models and newer support implicit caching" with no write cost; an +// explicit breakpoint is billed as a write (input price plus storage). So: never mark. +// OpenAI: automatic, no configuration. So: never mark. +// Stickiness: `session_id` in the body or the `x-session-id` header, at most 256 characters; +// "Sticky routing is skipped when you set `provider.order`" — this package sends no `provider` +// object at all, and must not start to. +// Usage: `prompt_tokens_details.cached_tokens` and `.cache_write_tokens`. +// https://openrouter.ai/docs/use-cases/usage-accounting: "The `usage: { include: true }` … +// parameters are deprecated and have no effect" — so it is not sent. +// +// OpenAI — automatic above 1,024 tokens (the OpenRouter page's summary of it); the body field +// `prompt_cache_key` groups requests that share a prefix. OpenRouter reads it as a session id +// when neither `session_id` nor the header is given. +// +// Checked against the live service, 2026-10-11, `anthropic/claude-sonnet-5.5` on OpenRouter, a +// three-turn `agent()` run: a mark on a `role: 'tool'` message's text part IS honoured (turn 2 +// read turn 1's 19,914-token prefix and wrote 72), and `prompt_tokens` INCLUDES both the read and +// the written tokens — which is why ./openai-wire subtracts both. +// +// The table is DATA: a new host or model family is a row. Unknown endpoints get NOTHING — a field +// a self-hosted server has never heard of is a 400 on some of them, and a mark is never worth that. + +/** + * A system prompt in two parts. `stable` is byte-identical across runs — rules, a knowledge-base + * index — and is what a cache shares BETWEEN runs; `run` is whatever differs per run and comes + * after it, so it never invalidates the stable half. A cache is a prefix match: one changed byte in + * `stable` (a timestamp, an unsorted list) is a full-price write on every call. + */ +export interface SystemParts { + readonly stable: string; + readonly run?: string; +} + +/** A plain string is the whole prompt as one stable part. */ +export type SystemPrompt = string | SystemParts; + +/** + * "Cache this request's prefix": present on a `GenerateRequest`, each provider marks the stable + * system part and the transcript's tail the way its wire wants — or not at all, where the host + * caches implicitly. Absent, nothing is marked and nothing is sent. + */ +export interface PromptCacheHint { + /** + * A key naming the runs that share a prefix, for hosts that route or shard by one (an + * `x-session-id` header on OpenRouter, `prompt_cache_key` on OpenAI). Never a secret, never + * personal data: it leaves the process in a header. + */ + readonly session?: string; +} + +/** OpenRouter's documented ceiling on a session id; OpenAI's key is held to the same one. */ +export const SESSION_KEY_MAX = 256; + +/** The one mark this package sends: the 5-minute TTL, which is the rate `costOf` prices. */ +export const CACHE_MARK = { type: 'ephemeral' } as const; + +/** Both parts as one text, stable first — for an estimate, and for a wire with one system string. */ +export function systemText(system: SystemPrompt | undefined): string { + if (system === undefined) return ''; + if (typeof system === 'string') return system; + return [system.stable, system.run ?? ''].filter((part) => part !== '').join('\n\n'); +} + +/** The parts, normalised: `''` for an absent half, so a wire never sends an empty block. */ +export function systemParts(system: SystemPrompt | undefined): Required { + if (system === undefined) return { stable: '', run: '' }; + if (typeof system === 'string') return { stable: system, run: '' }; + return { stable: system.stable, run: system.run ?? '' }; +} + +export type CacheWire = 'anthropic' | 'openai'; + +export interface PromptCacheRule { + readonly wire: CacheWire; + /** A hostname and every subdomain of it; `'*'` is any endpoint on this wire. */ + readonly host: string; + /** A model-id prefix; `''` is every model. */ + readonly model: string; + /** + * `blocks`: `cache_control` on Anthropic content blocks — the stable system block and the last + * block of the last two messages (three of the four breakpoints; the fourth is left free). + * `parts`: the same mark on chat-completions content PARTS — the stable system part and the tail + * message's last part (two breakpoints). + * `none`: the host caches by itself, or is unknown. + */ + readonly marks: 'blocks' | 'parts' | 'none'; + /** Where a `PromptCacheHint.session` goes. */ + readonly session: 'x-session-id' | 'prompt_cache_key' | 'none'; +} + +/** First match wins. */ +export const PROMPT_CACHE_RULES: readonly PromptCacheRule[] = [ + { wire: 'anthropic', host: '*', model: '', marks: 'blocks', session: 'none' }, + { + wire: 'openai', + host: 'openrouter.ai', + model: 'anthropic/', + marks: 'parts', + session: 'x-session-id', + }, + // Gemini, OpenAI, and everything else OpenRouter fronts: implicit or automatic. An explicit mark + // on Gemini turns a free implicit cache into a paid write. + { wire: 'openai', host: 'openrouter.ai', model: '', marks: 'none', session: 'x-session-id' }, + { wire: 'openai', host: 'api.openai.com', model: '', marks: 'none', session: 'prompt_cache_key' }, +]; + +const NO_RULE: Pick = { marks: 'none', session: 'none' }; + +/** OpenAI's own endpoint, which is what an `openAiProvider()` with no `baseUrl` talks to. */ +const DEFAULT_OPENAI_HOST = 'api.openai.com'; + +/** The rule for one request. Total: a base URL that does not parse matches no host. */ +export function promptCacheRule( + wire: CacheWire, + baseUrl: string | undefined, + model: string, +): Pick { + const host = baseUrl === undefined ? DEFAULT_OPENAI_HOST : hostOf(baseUrl); + const found = PROMPT_CACHE_RULES.find( + (rule) => + rule.wire === wire && + (rule.host === '*' || host === rule.host || host.endsWith(`.${rule.host}`)) && + model.startsWith(rule.model), + ); + return found ?? NO_RULE; +} + +function hostOf(baseUrl: string): string { + return URL.canParse(baseUrl) ? new URL(baseUrl).hostname.toLowerCase() : ''; +} + +/** + * What a declaration writes: `'auto'` marks every request of the call, `'off'` marks none, and the + * object form is `'auto'` plus a session key — a constant, or derived per call (per org, say). + */ +export type PromptCacheOption = + | 'auto' + | 'off' + | { readonly session: string | ((args: A) => string) }; + +/** The hint one call sends on every request, or `undefined` for none. */ +export function promptCacheHint( + option: PromptCacheOption | undefined, + fallback: 'auto' | 'off', + args: A, +): PromptCacheHint | undefined { + const chosen = option ?? fallback; + if (chosen === 'off') return undefined; + if (chosen === 'auto') return {}; + return { session: typeof chosen.session === 'string' ? chosen.session : chosen.session(args) }; +} diff --git a/packages/ai/src/provider.ts b/packages/ai/src/provider.ts index 1dd6f1aed..be25f3b5a 100644 --- a/packages/ai/src/provider.ts +++ b/packages/ai/src/provider.ts @@ -12,6 +12,9 @@ import type { AiFetch } from './fetch-seam'; import { resolveModel } from './model-resolve'; import type { Effort, ModelId, ThinkingMode } from './models'; import { modelSpec, reasoningBody } from './models'; +import type { PromptCacheHint, SystemPrompt } from './prompt-cache'; +import { promptCacheRule, systemText } from './prompt-cache'; +import { anthropicMessages, anthropicSystem } from './prompt-cache-blocks'; import { readSse } from './sse'; import type { LlmTool, LlmToolCall } from './tools'; import { @@ -65,7 +68,14 @@ export function messageText(message: AiMessage): string { export interface GenerateRequest { readonly model?: ModelId; - readonly system?: string; + /** One string, or a stable part and a per-run part (`./prompt-cache`) — two blocks on the wire. */ + readonly system?: SystemPrompt; + /** + * Ask the provider to cache this request's prefix: the stable system part and the transcript's + * tail, marked per wire and host by `PROMPT_CACHE_RULES`. Absent sends no mark. Not part of + * `promptCacheKey` — it changes what a call costs, never what it answers. + */ + readonly promptCache?: PromptCacheHint; readonly messages: readonly AiMessage[]; /** Enforced output ceiling. The model never sees it, so it can be cut off mid-answer. */ readonly maxTokens: number; @@ -328,13 +338,17 @@ export class AnthropicProvider implements Provider { const model = this.modelOf(request); // The blocks pass through untouched below — they ARE this wire's shapes — so this is the screen. assertMediaContent(request.messages, { provider: this.name, model }); + const marked = + request.promptCache !== undefined && + promptCacheRule('anthropic', this.config.baseUrl, model).marks === 'blocks'; const body: Record = { model, max_tokens: Math.min(request.maxTokens, modelSpec(model).maxOutput), - messages: request.messages.map((m) => ({ role: m.role, content: m.content })), + messages: anthropicMessages(request.messages, marked), ...reasoningBody(model, request.effort, request.thinking), }; - if (request.system !== undefined) body['system'] = request.system; + const system = anthropicSystem(request.system, marked); + if (system !== undefined) body['system'] = system; if (request.tools !== undefined && request.tools.length > 0) body['tools'] = request.tools; if (request.stopSequences !== undefined) body['stop_sequences'] = request.stopSequences; return body; @@ -439,7 +453,7 @@ const completionCeiling = (request: GenerateRequest): number => /** The prompt half alone — what the provider bills at the input rate. */ export function estimateInputTokens(request: GenerateRequest): number { const body = request.messages.map(messageText).join(' '); - const text = estimateTextTokens(body) + estimateTextTokens(request.system ?? ''); + const text = estimateTextTokens(body) + estimateTextTokens(systemText(request.system)); return text + mediaTokenEstimate(request.messages); } @@ -449,11 +463,15 @@ export function estimateInputTokens(request: GenerateRequest): number { * optimistic estimate is a ceiling one long completion walks through. */ export function estimateCost(request: GenerateRequest): Money { + // A hinted request can be billed as a cache WRITE end to end (turn one of a run), which is dearer + // than input — so that is the rate the ceiling is checked against. + const prompt = estimateInputTokens(request); + const write = request.promptCache !== undefined; return costOf(resolveModel('provider', request.model), { - inputTokens: estimateInputTokens(request), + inputTokens: write ? 0 : prompt, outputTokens: completionCeiling(request), cacheReadTokens: 0, - cacheWriteTokens: 0, + cacheWriteTokens: write ? prompt : 0, }); } diff --git a/packages/http/src/error-map-tier-4.ts b/packages/http/src/error-map-tier-4.ts index eea5941e7..e43ce887b 100644 --- a/packages/http/src/error-map-tier-4.ts +++ b/packages/http/src/error-map-tier-4.ts @@ -129,4 +129,8 @@ export const TIER_4_ERROR_STATUS = { X_PWA_PUSH_PAYLOAD_TOO_LARGE: 500, // @ultimat3/pwa — the push endpoint is not on a push service this app sends to X_PWA_PUSH_HOST_UNLISTED: 422, + // @ultimat3/ai — an agent repeated one tool call with one result and was stopped + X_AGENT_TOOL_LOOP: 500, + // @ultimat3/ai — an agent ran past its declared deadline and was stopped + X_AGENT_DEADLINE: 504, } satisfies Readonly>; diff --git a/wiki/Agents.md b/wiki/Agents.md index fabcacda2..e202d8448 100644 --- a/wiki/Agents.md +++ b/wiki/Agents.md @@ -59,6 +59,8 @@ The framework ships no default model and no catalogue row (25.0.0): every id is | How do I cancel an agent run when the user closes the tab? | `ctx.signal`. Over HTTP that signal is the **request deadline**, not the socket — a closed tab does not abort it `As of 2026-08` → [Cancellation](#cancellation) | | How do I run an agent over 2 million rows without blowing my budget? | `backfill()` for the sweep, `hive()` or `agentJob()` per page, `budget.tokensPerRun` on both, and a shared `BudgetStore` for the fleet-wide ceiling → [Millions of rows](#millions-of-rows) | | Do agent tool calls run in parallel? | Yes — every tool of **one turn**, concurrently and unbounded. Turns themselves are serial → [Tools of one turn run concurrently](#tools-of-one-turn-run-concurrently) | +| How do I stop paying full price for the same long prompt every turn? | Nothing — `agent()` asks the provider to cache each turn's prefix by default. Put text every run shares in `system().stable` → [Provider prompt caching](#provider-prompt-caching) | +| What stops an agent that loops, or hangs, with nobody watching? | `maxRepeatedToolCalls` (default 3) and `deadlineMs` → [Unattended runs](#unattended-runs) | | How do I show progress while an agent is thinking? | `onTurn`, called once per completed turn. There is **no `.stream()`** on an agent → [Progress](#progress-onturn) | ## A worked example @@ -125,6 +127,10 @@ export const supportAgent = agent({ | `maxTurns` | no | **8** | reaching it is `X_AGENT_MAX_TURNS`, never a partial answer | | `maxTokens` | no | **4,096** | completion ceiling **per turn**. The model never sees it | | `maxToolResultChars` | no | **4,000** | one tool result's ceiling; truncation says so in the transcript | +| `maxRepeatedToolCalls` | no | **3** | consecutive turns that may repeat the same calls **and** results; reaching it is `X_AGENT_TOOL_LOOP` → [Unattended runs](#unattended-runs) | +| `deadlineMs` | no | none | wall-clock ceiling for the whole run; passing it is `X_AGENT_DEADLINE` | +| `system` | no | — | system text loaded at run time, as `{ stable?, run? }` → [Provider prompt caching](#provider-prompt-caching) | +| `promptCache` | no | **`'auto'`** | `'auto'` \| `'off'` \| `{ session }` — asks the provider to cache each turn's prefix | | `budget` | no | unmetered | `tokensIn`, `costPerCall`, `tokensPerRun` | | `mcp` | no | not exposed | exposes the agent itself as a tool | | `onTurn` | no | — | one call per completed turn, awaited | @@ -259,6 +265,101 @@ That matters because the transcript **is** the request: a loop that keeps going | the next hop bounded by what is LEFT | nothing — both typed clients spread `traceHeaders()` before your own headers, and it sends the **remaining** budget as `x-request-timeout-ms`. A spent budget sends no header rather than `0`, because `0` reads one hop later as "the caller asked for nothing" | | a run the user can actually stop | run it as a job — `agentJob()` — and cancel it by id: `x jobs cancel --reason "user cancelled"` | +## Provider prompt caching + +Every turn re-sends the whole transcript, so a run with a long stable prefix — rules, a knowledge-base index — pays for that prefix once per turn. `agent()` asks the provider to cache it: **on by default**. + +```ts +import { action, t } from '@ultimat3/action'; +import { agent, definePrompt } from '@ultimat3/ai'; +import { can } from '@ultimat3/policy'; + +const readDocument = action({ + input: t.object({ id: t.string }), + output: t.object({ body: t.string }), + policy: can('kb:read'), + mcp: { expose: true, description: 'Read one knowledge-base document by id.' }, + handle: ({ input }) => ({ body: `body of ${input.id}` }), +}); + +// Your own loader. What it returns must be the same BYTES run to run: sorted, undated. +const knowledgeIndex = async (orgId: string): Promise => `INDEX for ${orgId}\n…`; + +const kbPrompt = definePrompt<{ question: string }>({ + id: 'kb', + version: '1.0.0', + system: 'Answer from the knowledge base only. Answer through the respond tool.', + template: 'Question: {{question}}', +}); + +export const kbAgent = agent({ + model: 'house-large', + input: t.object({ question: t.string, orgId: t.string }), + output: t.object({ answer: t.string }), + prompt: kbPrompt, // its static `system` starts the stable block + vars: ({ input }) => ({ question: input.question }), + // Text loaded at run time. `stable` joins the prompt's system in ONE block every run shares; + // `run` is a second block after it, for whatever differs. + system: async ({ input }) => ({ + stable: await knowledgeIndex(input.orgId), + run: `Asked on ${new Date().toISOString().slice(0, 10)}.`, + }), + // 'auto' (the default) | 'off' | { session } — a key for hosts that route by one + promptCache: { session: ({ input }) => `kb:${input.orgId}` }, + tools: [readDocument], + maxTurns: 10, + deadlineMs: 120_000, + policy: can('kb:ask'), + onTurn: ({ turn, usage, run }) => { + console.log(`turn ${turn}: read ${usage.cacheReadTokens}, wrote ${usage.cacheWriteTokens}`); + console.log(`run so far: ${run.cost.minor} ${run.cost.currency}`); + }, +}); +``` + +| Turn | What the provider is asked | What it bills | +|---|---|---| +| 1 | cache the stable system block, and the transcript up to its tail | the prefix as a cache **write** | +| N+1 | the same, with the mark moved to the new tail | turn N's prefix as a cache **read**, only the new tail as a write | +| 1 of the next run | the same stable block, byte for byte | the stable block as a read, while the cache entry lives (5 minutes, refreshed by each read) | + +**What is sent depends on the wire and the host** — `PROMPT_CACHE_RULES` in [`prompt-cache.ts`](https://github.com/developerz-ai/ultimate/blob/main/packages/ai/src/prompt-cache.ts), first match wins. `As of 2026-10`: + +| Provider | Endpoint | Model id | Mark | `session` goes | +|---|---|---|---|---| +| `anthropicProvider()` | any | any | `cache_control: { type: 'ephemeral' }` on the stable system block and the last block of the last two messages — 3 of the 4 breakpoints | nowhere | +| `openAiProvider()` | `openrouter.ai` | `anthropic/…` | the same mark on the stable system **part** and the tail message's last part — 2 breakpoints | `x-session-id` header | +| `openAiProvider()` | `openrouter.ai` | anything else | **none** — Gemini and OpenAI cache implicitly, and an explicit mark on Gemini turns a free cache into a paid write | `x-session-id` header | +| `openAiProvider()` | `api.openai.com` (the default) | any | none — caching is automatic | `prompt_cache_key` body field | +| `openAiProvider()` | any other `baseUrl` | any | none | nowhere | + +| Rule | Detail | +|---|---| +| The stable block must be the same **bytes** | a cache is a prefix match. A timestamp, a request id or an unsorted list in `stable` is a full-price write on every call — put it in `run` | +| Below the model's minimum nothing is cached | 512 to 4,096 tokens depending on the model (`ModelSpec.cacheMinimumTokens`); a shorter prefix is processed uncached, with no error | +| `session` is a routing key, not a secret | 1 to 256 printable ASCII characters, else `X_AI_REQUEST_INVALID`. It leaves the process in a header | +| No routing preference is ever sent | OpenRouter skips sticky routing when a request orders its providers, and a cache lives on one upstream | +| `llm()` is **off** by default | `promptCache: 'auto'` on an `llm()` that is called many times with one long `system`. A one-shot call that marks its prefix pays the write premium for a cache nothing reads | +| A direct `gateway.generate()` call | `system: { stable, run }` and `promptCache: {}` on the request — the same two fields the loop sets | + +**Reading it back.** `onTurn`'s `usage` carries `cacheReadTokens` and `cacheWriteTokens` per turn, `run` carries the running total (`run.usage`, `run.cost`), and the `agent.turn` span event carries `llm.cache.read_tokens` / `llm.cache.write_tokens`. A run whose later turns read nothing has an unstable prefix. + +**What it costs.** `costOf` prices a read at the model row's `cacheReadPerMillion` (absent: 0.1x input) and a write at `cacheWritePerMillion` (absent: 1.25x input). A token budget (`tokensPerRun`, `tokensIn`, the gateway's actor and org ceilings) counts **every** token the provider processed — input, output, cache read and cache write, each as one — so caching lowers what a run costs, not how many tokens it is allowed. The pre-flight estimate of a marked request prices the whole prompt as a cache write, the dearest it can be billed, so `costPerCall` holds on turn one. + +Measured, 2026-10-11, one three-turn run against OpenRouter with a 19,914-token stable prefix: turn 1 wrote 19,914 tokens and read 0; turn 2 read 19,914 and wrote 72; turn 3 read 19,986 and wrote 68. + +## Unattended runs + +Three bounds stop a run nobody is watching, beside `maxTurns` and `budget.tokensPerRun`: + +| Bound | Default | Reaching it | +|---|---|---| +| `maxRepeatedToolCalls` | **3** | `X_AGENT_TOOL_LOOP` — that many consecutive turns made the **same tool calls and read the same results**. A polled tool whose answer changes is progress and is left alone | +| `deadlineMs` | none | `X_AGENT_DEADLINE` — the run was still going; the model call in flight is aborted on the socket, a tool still running is no longer waited for, and no further turn starts | +| `maxToolResultChars` | **4,000** | not an error — the result is truncated, and says so in the transcript | + +A caller that goes away mid-call is `X_ABORTED`, not the transport failure the aborted socket would otherwise report — that one is retryable, and a job would re-run a run nobody is waiting for. + ## Progress (`onTurn`) A 90-second agent run emits nothing until it returns. `onTurn` is what a progress indicator, a per-turn spend line and a transcript log read. @@ -267,7 +368,8 @@ A 90-second agent run emits nothing until it returns. `onTurn` is what a progres onTurn: ({ turn, maxTurns, model, toolCalls, stopReason, usage, cost }) => { // turn is 1-based, so `turn === maxTurns` is the last one this run will take // toolCalls are names, in the order the model emitted them — `respond` is not one - // cost is THIS turn's alone, never the run's running total + // usage and cost are THIS turn's alone: inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens + // the event also carries `run` — the running total, this turn included: run.usage, run.cost }, ``` @@ -468,6 +570,8 @@ Similar prompts do not have similar answers once the answer depends on what `loo |---|---|---| | `X_AGENT_TOOL_UNEXPOSED` | a `tools` entry declares no `mcp` block, or `mcp: { expose: false }`. Raised at **declaration** | add `mcp: { expose: true, description }` to the action `cause` names, or drop it from `tools` | | `X_AGENT_MAX_TURNS` | the loop used every turn calling tools and never answered | tell the template when to stop and answer through the respond tool, then bump its version | +| `X_AGENT_TOOL_LOOP` | consecutive turns made the same tool calls and read the same results | `maxRepeatedToolCalls: 5` on the agent, only for a tool that really is polled; otherwise tell the template what to do when the result does not change, then bump its version | +| `X_AGENT_DEADLINE` | the run outlived its declared `deadlineMs` | `deadlineMs: 300_000` on the agent, once a trace shows the run converging; otherwise lower `maxTurns` | | `X_LLM_OUTPUT_INVALID` | every turn's answer failed the `output` schema | describe the shape in the template and bump its version, or widen `output` | | `X_LLM_REFUSED` | the provider's classifiers declined; `stopDetails.category` names why | set `model:` to the more capable one the thrown `fix:` names, or edit the template and bump its version | | `X_LLM_TRUNCATED` | a turn hit `maxTokens` before finishing | raise `maxTokens:` on the declaration, or drop fields from `output` | @@ -489,6 +593,8 @@ Full list: [Error codes](Error-Codes). - A budget refuses, never truncates, and the ceiling holds under parallelism because the reservation debits before the call. - `ctx.signal` unwinds the run at every turn boundary, before every tool batch, and on the socket. - `onTurn` observes and cannot steer. There is no `.stream()` on an agent. +- Every turn asks the provider to cache its prefix unless `promptCache: 'off'`; what is sent is a row in `PROMPT_CACHE_RULES`, never an `if`. +- A run that repeats itself is stopped (`X_AGENT_TOOL_LOOP`); a run with a `deadlineMs` is stopped at it (`X_AGENT_DEADLINE`). - A hive reports in split order with three member arms; an empty split is an error. - `idempotencyKey` dedupes the enqueue, never the attempt — every tool an agent may call must be idempotent, and nothing checks it for you. diff --git a/wiki/Error-Codes.md b/wiki/Error-Codes.md index c66e93f3e..e0c0ac3a1 100644 --- a/wiki/Error-Codes.md +++ b/wiki/Error-Codes.md @@ -707,6 +707,8 @@ Three of them are raised at `notifier(...)`, while the app's modules load, so th | `X_MCP_CONFIRMATION_CONTESTED` | identical concurrent MCP calls kept racing one confirmation | one agent sent the same call concurrently and the open row kept being consumed | `tools/call` — retry once, after the concurrent identical calls finish: send one at a time | | `X_MCP_CONFIRMATION_ARGUMENTS_MISMATCH` | an MCP confirmation approval did not carry the arguments the agent sent | approve was sent without arguments, with different arguments than view returned, or on a row keyed before a signing-secret rotation | `confirmations({ id, decision: 'view' })` — then approve with exactly the arguments it returns | | `X_AI_MODEL_UNRESOLVED` | a model call named no model | neither the declaration, its prompt, nor providerGateway({ defaultModel }) named one — the framework chooses no model for an app | `llm({ ..., model: modelId })` — or model: modelId in its definePrompt, or providerGateway({ ..., defaultModel: modelId }); register the id with registerModel | +| `X_AGENT_TOOL_LOOP` | an agent repeated one tool call with one result and was stopped | an agent() asked for the same tool calls with the same arguments and read the same results on consecutive turns, so no later turn can differ. Stopped at maxRepeatedToolCalls (default 3) instead of spending every remaining turn re-sending a growing transcript for the same non-answer | `maxRepeatedToolCalls: 5` — on the agent, only for a tool that is polled and answers identically while it waits; otherwise tell the template what to do when a tool result does not change, then bump its version | +| `X_AGENT_DEADLINE` | an agent ran past its declared deadline and was stopped | an agent() declared `deadlineMs` and the run was still going when it elapsed: the model call in flight was aborted and no further turn or tool was started. The run has no answer; tools that already ran keep their effects | `deadlineMs: 300_000` — on the agent, only once a trace shows the run converging; otherwise lower `maxTurns`, or shorten the slow tool the `agent.turn` span events name | ## Scraping diff --git a/wiki/MCP-And-AI.md b/wiki/MCP-And-AI.md index 5ee769433..e8eda966e 100644 --- a/wiki/MCP-And-AI.md +++ b/wiki/MCP-And-AI.md @@ -341,6 +341,7 @@ export const summarize = llm({ | Cost + token accounting | per call, per tenant, per prompt version; exceeding `budget` throws before spending | | Retries | typed on provider errors; rate limits back off, content refusals do not retry | | Caching | semantic cache from [Caching and invalidation](Caching-And-Invalidation) | +| Provider prompt caching | the provider keeps a request's stable prefix; on by default for `agent()`, off for `llm()` → [Agents](Agents#provider-prompt-caching) | | Tracing | one OTel span per call with model, tokens, cache hit, prompt version | | Fallback | ordered provider list per model; the provider that answered is on the span as `llm.provider`, never silent. A refusal names a more capable model for the declaration to adopt — models are never swapped at runtime | | Money | `Money = { minor, currency }` — never a float → [Money](Money) | From 2bb42d82539ffbf01ccc0e8baca7ca7a484f1b3b Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 09:43:53 -0500 Subject: [PATCH 2/3] fix(ai): agent stops are terminal for jobs; tools hear the run deadline; the write-rate estimate only where a provider marks; alternating tool cycles are caught; no cache mark on a blank block or past four Review follow-ups to the prompt-caching change: - X_AGENT_TOOL_LOOP and X_AGENT_DEADLINE are registered terminal, so an agentJob dead-letters on attempt one instead of buying the same run per attempt. - The run deadline rides each tool's own ctx.signal. - estimateCost prices a cache write only when a provider serving the model would mark the request (Provider.marksPromptCache), at the dearer of write and input. - The loop guard detects cycles of one to three tool turns. - The Anthropic wire never marks an empty or whitespace-only text block and counts a caller's own marks against the four first. - A prompt_cache_key over 64 characters is sent to OpenAI as a hash of itself. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +- docs/history/ai.md | 11 + packages/ai/CLAUDE.md | 22 +- packages/ai/README.md | 19 +- packages/ai/src/agent-errors.ts | 16 +- packages/ai/src/agent-guards.test.ts | 93 +++++++- packages/ai/src/agent-guards.ts | 46 +++- packages/ai/src/agent-job.test.ts | 64 ++++- packages/ai/src/agent.ts | 5 +- packages/ai/src/budget.ts | 5 +- packages/ai/src/errors-retry.test.ts | 30 ++- packages/ai/src/errors-retry.ts | 6 + packages/ai/src/gateway.ts | 11 +- packages/ai/src/index.ts | 8 +- packages/ai/src/openai-body.ts | 22 +- packages/ai/src/openai-messages.ts | 13 +- packages/ai/src/openai-provider.ts | 5 + packages/ai/src/prompt-cache-blocks.test.ts | 250 ++++++++++++++++++++ packages/ai/src/prompt-cache-blocks.ts | 101 ++++++-- packages/ai/src/prompt-cache-fixture.ts | 86 +++++++ packages/ai/src/prompt-cache.test.ts | 225 +++--------------- packages/ai/src/prompt-cache.ts | 18 +- packages/ai/src/provider.ts | 44 ++-- wiki/Agents.md | 17 +- wiki/Error-Codes.md | 4 +- 25 files changed, 837 insertions(+), 294 deletions(-) create mode 100644 packages/ai/src/prompt-cache-blocks.test.ts create mode 100644 packages/ai/src/prompt-cache-fixture.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 37961c9a6..d40edeb0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,17 +13,17 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major Tier 2 — http. Tier 4 — ai. - **Provider prompt caching: an `agent()` no longer pays input price for its stable prefix on every turn.** Every turn re-sends the whole transcript, and nothing asked a provider to cache it. A `GenerateRequest` now takes `system: { stable, run }` (two blocks; the stable one first and byte-identical across runs) and `promptCache: { session? }`, and each provider marks the stable system part and the transcript's tail the way its wire and host want — `PROMPT_CACHE_RULES` in `@ultimat3/ai`, a table, sources quoted in `prompt-cache.ts`: `cache_control: { type: 'ephemeral' }` on Anthropic content blocks (three of the four breakpoints), the same mark on content parts for an `anthropic/…` model on OpenRouter (two), and **no** mark for any other model there or on OpenAI, where caching is implicit and an explicit mark on Gemini is a paid write. A session key travels as OpenRouter's `x-session-id` header or OpenAI's `prompt_cache_key`; an endpoint the table does not name is sent nothing. `agent({ system, promptCache })` and `llm({ promptCache })` are the declaration's side: `'auto' | 'off' | { session }`, and `system()` returns the text loaded at run time as `{ stable?, run? }`. -- **Default: `agent()` caches (`promptCache: 'auto'`), `llm()` does not (`'off'`).** This is the one behaviour that changes for a caller who passes nothing: an agent's requests to an Anthropic model now carry `cache_control` marks (so `system` is sent as blocks or parts, not a string), a first turn is billed as a cache write (1.25x input on Anthropic) and later turns as reads, and the pre-flight cost estimate of a marked request prices the prompt as a write — a `budget.costPerCall` set within 25% of an agent's prompt cost now refuses. `promptCache: 'off'` restores the old request byte for byte. A one-shot `llm()` is unchanged. +- **Default: `agent()` caches (`promptCache: 'auto'`), `llm()` does not (`'off'`).** This is the one behaviour that changes for a caller who passes nothing: an agent's requests to an Anthropic model now carry `cache_control` marks (so `system` is sent as blocks or parts, not a string), a first turn is billed as a cache write (1.25x input on Anthropic) and later turns as reads, and the pre-flight cost estimate prices the prompt as a write **where a provider serving the model would mark the request** (`Provider.marksPromptCache`), at the dearer of the write and input rates — so a `budget.costPerCall` set within 25% of an agent's prompt cost now refuses for an Anthropic model, and is unchanged for Gemini, OpenAI and any endpoint the table does not name. `promptCache: 'off'` restores the old request byte for byte. A one-shot `llm()` is unchanged. A mark is never put on an empty or whitespace-only text block, and marks a raw request already carries count against the vendor's four first. A session key over 64 characters is sent to OpenAI as a hash of itself; OpenRouter's header takes up to 256 and refuses more. - **Cache usage is reported per turn and per run.** The chat-completions reader takes `prompt_tokens_details.cache_write_tokens` as well as `cached_tokens` out of `prompt_tokens` (which includes both) and reports them as `cacheWriteTokens` / `cacheReadTokens`; it reported a write as full-price input before. `onTurn`'s event gains `run` — the running `usage` and `cost`, priced once from the summed usage — and the `agent.turn` span event gains `llm.cache.read_tokens` / `llm.cache.write_tokens`. Prices are the model row's existing `cacheReadPerMillion` / `cacheWritePerMillion`; a token budget counts read and written tokens one for one, as it always has. -- **`maxRepeatedToolCalls` — an agent that repeats itself is stopped (`X_AGENT_TOOL_LOOP`, 500).** That many consecutive turns making the same tool calls **and** reading the same results end the run. **Default 3, on for every agent**: a loop that used to end in `X_AGENT_MAX_TURNS` or `X_AI_BUDGET_EXCEEDED` after repeating one call now ends earlier with this code. A polled tool whose answer changes is not a repeat. -- **`deadlineMs` — a wall-clock ceiling for one agent run (`X_AGENT_DEADLINE`, 504).** No default. Passing it aborts the model call in flight on its socket, stops waiting for a tool still running, and starts no further turn. -- `@ultimat3/ai` exports `PROMPT_CACHE_RULES`, `promptCacheRule`, `systemText`, `SESSION_KEY_MAX`, `AgentToolLoopError`, `AgentDeadlineError` and the types `SystemPrompt`, `SystemParts`, `PromptCacheHint`, `PromptCacheOption`, `PromptCacheRule`, `CacheWire`, `AgentSystem`, `AgentRunUsage`. +- **`maxRepeatedToolCalls` — an agent that repeats itself is stopped (`X_AGENT_TOOL_LOOP`, 500).** A cycle of one to three tool turns — the same calls, arguments, results and failures — gone round that many times ends the run: the same turn over and over, or A,B,A,B…. **Default 3, on for every agent**: a loop that used to end in `X_AGENT_MAX_TURNS` or `X_AI_BUDGET_EXCEEDED` after repeating itself now ends earlier with this code. A polled tool whose answer changes is not a repeat; a repair turn neither counts nor resets. Registered `terminal`, so an `agentJob()` dead-letters it on the first attempt. +- **`deadlineMs` — a wall-clock ceiling for one agent run (`X_AGENT_DEADLINE`, 504).** No default. Passing it aborts the model call in flight on its socket, fires every running tool's own `ctx.signal`, and starts no further turn. A tool that ignores its signal is no longer waited for but still runs to its end. Registered `terminal` for jobs. +- `@ultimat3/ai` exports `PROMPT_CACHE_RULES`, `promptCacheRule`, `systemText`, `SESSION_KEY_MAX`, `OPENAI_CACHE_KEY_MAX`, `AgentToolLoopError`, `AgentDeadlineError` and the types `SystemPrompt`, `SystemParts`, `PromptCacheHint`, `PromptCacheOption`, `PromptCacheRule`, `CacheWire`, `AgentSystem`, `AgentRunUsage`. ### Fixed Tier 4 — ai. -- **A caller that goes away during an agent's model call is `X_ABORTED`.** The aborted socket surfaced as `X_AI_PROVIDER_UNAVAILABLE`, the one code the package classifies retryable, so a job retried a run its caller had abandoned. +- **A caller that goes away during an agent's model call is `X_ABORTED`.** The aborted socket surfaced as `X_AI_PROVIDER_UNAVAILABLE`, which the package classifies `retryable`, so an HTTP client was told to come back for a run it had itself abandoned. `X_ABORTED` is core's and unclassified: it reads `terminal` on the wire, and in a job the attempt count still decides — nothing about a drain or an attempt timeout changes, jobs settles those off its own signal. ## 27.8.1 - 2026-10-11 diff --git a/docs/history/ai.md b/docs/history/ai.md index 79fea67e3..338135c70 100644 --- a/docs/history/ai.md +++ b/docs/history/ai.md @@ -733,3 +733,14 @@ until 2026-08, naming a tool no catalog contained (`llm.test.ts`, `agent.test.ts `orgless` case `query`'s cache uses; `RetryPolicy` / `DEFAULT_GATEWAY_RETRY` keep the field names an app writes; the status tables and the OpenAI `estimatedUsage` fallback are the differences `provider-parity.test.ts` asserts between the two wire formats. +- **Review follow-ups (same change, second commit).** The two agent stops are registered + `terminal` because an unclassified code keeps a job's attempt count: an `agentJob` with + `attempts: 3` bought a repeating run three times. `X_ABORTED` was deliberately NOT classified + in core: jobs raises and reads that code itself (a fenced step write, an attempt timeout), and + a registered `terminal` would dead-letter those on attempt one. The write-rate estimate is + gated on `Provider.marksPromptCache` because `estimateCost` cannot see a host, and an estimate + that rose 25% for a Gemini agent refused calls a tight `costPerCall` had allowed. The loop + guard looks for cycles of up to three turns because A,B,A,B survived fifty turns under a + consecutive-only comparison. OpenAI's `prompt_cache_key` maximum could not be confirmed (its + guide and SDK types state none; the API reference answered 403), so a long key is hashed, not + refused: a refusal on an unverified limit would fail calls the endpoint may accept. diff --git a/packages/ai/CLAUDE.md b/packages/ai/CLAUDE.md index 909090319..d1ddf26b0 100644 --- a/packages/ai/CLAUDE.md +++ b/packages/ai/CLAUDE.md @@ -129,8 +129,8 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe default provider. - **`llm()` forwards `ctx.signal`** onto `GenerateRequest`; `.stream()` inherits it. - **`.stream()` is the SAME action**, marked with an ambient sink: UNVALIDATED text, then one `done` - with the value that satisfied `output`; no repair turn (`X_LLM_STREAM_INVALID`), no `respond` tool; - budget reserved before the provider, reconciled at `done`; a cache hit yields `done` alone. LAZY. + with the validated value; no repair turn (`X_LLM_STREAM_INVALID`), no `respond` tool; budget + reserved before the provider, reconciled at `done`. LAZY. - **`configureAi({ redact })` is the one seam between `vars()` and the provider**; a `Secret` in `vars()` is `X_AI_PROMPT_SECRET` regardless. The `llm.redacted` span attribute records it. - **Fallback is across PROVIDERS serving one model, never across models**; `GenerateResult.provider` @@ -156,8 +156,8 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe - **The gateway's reads of a provider's throw are total**: `isRetryable` fails closed; the line goes through `renderThrowable`. - **`X_AI_PROVIDER_UNAVAILABLE` is the one code classified `retryable`** (`AI_ERROR_RETRY`); - `AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Every other code - stays UNREGISTERED on purpose. + `AiProviderUnavailableError({ unserved: true })` carries a per-instance `terminal`. Every other + code but the two agent stops stays UNREGISTERED on purpose. - **The retry SCHEDULE is core's (`backoffDelay`, `isRetryableStatus`); the LOOP is this package's** — core's `retry()` retries unclassified throws, so it would retry a 400 (`gateway-backoff.test.ts`). - Every non-2xx and every in-band `error` frame is `AiTransportError` with a real `status`. @@ -257,14 +257,16 @@ exposure, the whole input schema (an idempotent action's reserved `idempotencyKe - **`GenerateRequest.promptCache` is the one switch; what it sends is a ROW in `PROMPT_CACHE_RULES`** (wire, host, model prefix → marks, session carrier), never an `if`. An unnamed endpoint gets - NOTHING. At most 3 `cache_control` (the vendor's cap is 4); never a `provider` routing object. + NOTHING. At most 3 `cache_control`, a caller's own counted first (cap 4), never on a blank text + block; never a `provider` routing object. - **`agent()` defaults `'auto'`, `llm()` `'off'`.** The stable system part (`prompt.system` + - `system().stable`) is sent FIRST and byte-stable; the mark follows the transcript's tail. The hint - is not in `promptCacheKey`. + `system().stable`) is sent FIRST; the mark follows the tail. The hint is not in `promptCacheKey`. - **Usage**: chat-completions `prompt_tokens` INCLUDES read and written tokens; both are subtracted, - clamped. A budget debits all four counters; `estimateCost` prices a hinted prompt as a WRITE. -- **`maxRepeatedToolCalls` compares calls AND results**; `deadlineMs` has no default and rides the - request's signal; a caller abort mid-call is `X_ABORTED`, never the transport's code. + clamped. A budget debits all four counters; the estimate prices a WRITE only where a provider + would mark (`marksPromptCache`), at the dearer rate. +- **`maxRepeatedToolCalls` compares calls AND results over cycles of 1–3 turns**; `deadlineMs` has no + default and rides the request's AND each tool's signal; both stops are `terminal` + (`errors-retry.ts`). ## Invariants — evals, retrieval, fix lines diff --git a/packages/ai/README.md b/packages/ai/README.md index 35ac613b4..b5df29712 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -435,8 +435,9 @@ sub-agent in its own `tools` and the sub-agent runs under the same actor, throug | `onTurn` reports each completed turn as it happens (and an `agent.turn` span event, always) | a 90-second run emitted nothing until it returned. Observation only — it cannot steer the loop, see the transcript or reach the actor — and a throw from it fails the run rather than being swallowed | | a tool that is not `mcp: { expose: true }` is `X_AGENT_TOOL_UNEXPOSED` **at declaration** | a silently dropped tool reads as offered and is not; `isMcpExposed` is the one predicate, so an in-app agent and an external MCP client see the same catalogue | | running out of turns is `X_AGENT_MAX_TURNS`, never a partial answer | a half-finished transcript returned as a result is working notes presented as a decision | -| `maxRepeatedToolCalls` (default 3) consecutive turns with the same calls **and** results is `X_AGENT_TOOL_LOOP` | such a turn can only be followed by another like it; a polled tool whose answer changes is left alone | -| `deadlineMs` (no default) is the run's wall-clock ceiling: `X_AGENT_DEADLINE` | the in-flight model call is aborted on the socket; a caller that leaves first is still `X_ABORTED` | +| a cycle of 1 to 3 tool turns — same calls, arguments, results and failures — gone round `maxRepeatedToolCalls` (default 3) times is `X_AGENT_TOOL_LOOP` | such a turn can only be followed by another like it; a changed result is progress; a repair turn neither counts nor resets | +| `deadlineMs` (no default) is the run's wall-clock ceiling: `X_AGENT_DEADLINE` | the in-flight model call is aborted on the socket and every tool's `ctx.signal` fires; a tool that ignores it still finishes late. A caller that leaves first is still `X_ABORTED` | +| both stops are registered `terminal` | an `agentJob()` dead-letters on attempt one instead of buying the same run per attempt | | every turn asks the provider to cache its prefix (`promptCache`, default `'auto'`) | the transcript IS the request, so an uncached run pays for its stable prefix once per turn — [below](#provider-prompt-caching) | | `budget.tokensPerRun` caps the **whole run** | a single call is bounded by `maxTokens`; a loop is bounded by nothing until this is set | | a tool result is truncated, and says so | the transcript IS the request, so an untruncated result is re-billed once per remaining turn | @@ -496,8 +497,18 @@ Usage: the chat-completions reader takes `prompt_tokens_details.cached_tokens` a `.cache_write_tokens` out of `prompt_tokens` (which includes both) and reports them as `cacheReadTokens` / `cacheWriteTokens`; a missing field is zero. `costOf` prices them at the model row's `cacheReadPerMillion` / `cacheWritePerMillion` (absent: 0.1x / 1.25x input). A token budget -counts all four counters, one token each; `estimateCost` prices a hinted request's prompt as a -write. No `provider` routing object and no `usage: { include }` is ever sent. +counts all four counters, one token each. The gateway's pre-flight estimate prices the prompt as a +write only when a provider serving the model would mark it (`Provider.marksPromptCache`), at the +dearer of the write and input rates. No `provider` routing object and no `usage: { include }` is +ever sent. + +| Limit | Detail | +|---|---| +| `run.cost` vs the ledger | `AgentTurn.run.cost` is summed usage priced once; `gateway.spent()` adds each turn's cost, rounded up per turn (input and output halves), so it can be higher by under 2 minor units per turn. Budgets are enforced on the ledger; report `run.cost` | +| the Anthropic row matches any host | an `anthropicProvider({ baseUrl })` behind a proxy still sends marks | +| more than about 9 tool calls in one turn, on OpenRouter | only the tail is marked there and the provider looks back 20 blocks, so that turn's transcript is rewritten, not read | +| a mark never lands on an empty or whitespace-only text block, and never makes a fifth | it moves back a block or is dropped; a raw caller's own marks are counted first | +| `session` | OpenRouter header: 1 to 256 printable ASCII, else `X_AI_REQUEST_INVALID`. OpenAI `prompt_cache_key`: over 64 characters it is sent as `h:` + 62 hex of its SHA-256 | ## `hive()` — many members, one action diff --git a/packages/ai/src/agent-errors.ts b/packages/ai/src/agent-errors.ts index 4e1c17d23..1e75bf50e 100644 --- a/packages/ai/src/agent-errors.ts +++ b/packages/ai/src/agent-errors.ts @@ -13,14 +13,22 @@ import { UltimateError } from '@ultimat3/core'; * same non-answer over a transcript that is one turn longer. */ export class AgentToolLoopError extends UltimateError { - constructor(input: { agent: string; repeats: number; tools: readonly string[] }) { + constructor(input: { + agent: string; + /** Laps of the cycle. */ + repeats: number; + /** Turns in one lap: 1 is the same turn over and over, 2 is A,B,A,B. */ + cycle: number; + tools: readonly string[]; + }) { super({ code: 'X_AGENT_TOOL_LOOP', cause: - `agent "${input.agent}" made the same tool calls (${input.tools.join(', ')}) and read the ` + - `same results ${input.repeats} turns in a row`, + input.cycle === 1 + ? `agent "${input.agent}" made the same tool calls (${input.tools.join(', ')}) and read the same results ${input.repeats} turns in a row` + : `agent "${input.agent}" went round the same ${input.cycle} tool turns (${input.tools.join(', ')}), reading the same results, ${input.repeats} times in a row`, fix: 'maxRepeatedToolCalls: 5 # on the agent, ONLY for a tool that is polled and answers identically while it waits — otherwise tell the template what to do when a tool result does not change, then bump its version', - meta: { agent: input.agent, repeats: input.repeats, tools: input.tools }, + meta: { agent: input.agent, repeats: input.repeats, cycle: input.cycle, tools: input.tools }, }); } } diff --git a/packages/ai/src/agent-guards.test.ts b/packages/ai/src/agent-guards.test.ts index 67470adba..e3726d207 100644 --- a/packages/ai/src/agent-guards.test.ts +++ b/packages/ai/src/agent-guards.test.ts @@ -4,7 +4,7 @@ */ import { describe, expect, test } from 'bun:test'; -import { ctxOf, userActor } from '@ultimat3/core'; +import { ctxOf, useContext, userActor } from '@ultimat3/core'; import { allow } from '@ultimat3/policy'; import { t } from '@ultimat3/schema'; import { agent } from './agent'; @@ -152,6 +152,71 @@ describe('a loop that repeats itself is stopped', () => { expect(same).toHaveLength(3); }); + test('a short alternating cycle is a loop too: A,B,A,B,A,B stops at the third lap', async () => { + const seen = model((turn) => ({ name: 'status', input: { id: turn % 2 === 0 ? 'b' : 'a' } })); + const run = declare( + [tool('status', ({ input }) => Promise.resolve(`pending ${JSON.stringify(input)}`))], + { + maxTurns: 50, + }, + ); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + meta: { repeats: 3, cycle: 2 }, + }); + // Three laps of two turns, not fifty turns. + expect(seen).toHaveLength(6); + }); + + test('a cycle of three is caught; a cycle of four is past the window and left to maxTurns', async () => { + const three = model((turn) => ({ name: 'status', input: { id: `s${turn % 3}` } })); + const run = declare([tool('status', () => Promise.resolve('pending'))], { maxTurns: 50 }); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + meta: { cycle: 3 }, + }); + expect(three).toHaveLength(9); + + const four = model((turn) => ({ name: 'status', input: { id: `s${turn % 4}` } })); + const wide = declare([tool('status', () => Promise.resolve('pending'))], { maxTurns: 20 }); + await expect(wide({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_MAX_TURNS', + }); + expect(four).toHaveLength(20); + }); + + test('an alternating cycle whose results keep changing is progress', async () => { + let ran = 0; + const seen = model((turn) => + turn <= 8 + ? { name: 'status', input: { id: turn % 2 === 0 ? 'b' : 'a' } } + : { name: 'respond', input: { answer: 'done' } }, + ); + const run = declare( + [ + tool('status', () => { + ran += 1; + return Promise.resolve(`pending ${ran}`); + }), + ], + { maxTurns: 12 }, + ); + expect(await run({ id: 'x' }, { ctx: ctx() })).toEqual({ answer: 'done' }); + expect(seen).toHaveLength(9); + }); + + test('a repair turn between identical tool turns does not reset the count', async () => { + // tool, tool, a wrong-shaped answer (a repair turn, no tool ran), tool → the third identical. + const seen = model((turn) => + turn === 3 ? { name: 'respond', input: { wrong: true } } : status(), + ); + const run = declare([tool('status', () => Promise.resolve('pending'))]); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_TOOL_LOOP', + }); + expect(seen).toHaveLength(4); + }); + test('the limit is the declaration’s, and a limit that could never pass a first turn is refused', async () => { const seen = model(status); const run = declare([tool('status', () => Promise.resolve('pending'))], { @@ -192,6 +257,32 @@ describe('a run has a deadline when it declares one', () => { expect(seen).toHaveLength(1); }); + test('a tool reads the deadline on its own ctx.signal, so a cooperative tool stops with the run', async () => { + const seen = model(status); + let heard: 'no signal' | 'waiting' | 'aborted' = 'no signal'; + const run = declare( + [ + tool('status', () => { + // What a real `action()` tool is handed as `ctx.signal`. + const { signal } = useContext(); + heard = 'waiting'; + return new Promise((resolve) => { + signal.addEventListener('abort', () => { + heard = 'aborted'; + resolve('stopped'); + }); + }); + }), + ], + { deadlineMs: 30 }, + ); + await expect(run({ id: 'x' }, { ctx: ctx() })).rejects.toMatchObject({ + code: 'X_AGENT_DEADLINE', + }); + expect(heard as string).toBe('aborted'); + expect(seen).toHaveLength(1); + }); + test('the caller leaving first is still X_ABORTED, and no deadline means no timer', async () => { const controller = new AbortController(); const hung = model(status, true); diff --git a/packages/ai/src/agent-guards.ts b/packages/ai/src/agent-guards.ts index 8ead8576c..e9acb70fb 100644 --- a/packages/ai/src/agent-guards.ts +++ b/packages/ai/src/agent-guards.ts @@ -1,7 +1,7 @@ // Single responsibility: the two guards that let an `agent()` run with nobody watching — the // repeated-turn detector and the per-run deadline. Apart from ./agent so the loop reads as a loop. -import { fingerprint } from '@ultimat3/core'; +import { fingerprint, withChildContext } from '@ultimat3/core'; import { AgentDeadlineError, AgentToolLoopError } from './agent-errors'; import type { LlmToolCall, LlmToolResult } from './tools'; @@ -12,30 +12,45 @@ import type { LlmToolCall, LlmToolResult } from './tools'; */ export const DEFAULT_MAX_REPEATED_TOOL_CALLS = 3; +/** The longest cycle looked for: A,B,C,A,B,C. A longer one is `maxTurns`' to stop. */ +const MAX_CYCLE = 3; + /** - * Called once per tool turn, with what the model asked for and what it was told. Throws - * `X_AGENT_TOOL_LOOP` on the `limit`-th consecutive turn whose calls AND results are the ones - * before it. A fingerprint, not the text: one result can be megabytes before it is truncated. + * Called once per TOOL turn, with what the model asked for and what it was told. Throws + * `X_AGENT_TOOL_LOOP` when the most recent tool turns are one cycle of up to `MAX_CYCLE` turns + * gone round `limit` times — the same turn `limit` times, or A,B,A,B… — where "the same" is the + * calls, their arguments, the results AND whether each failed. A changed result is progress and + * breaks the cycle. A fingerprint, not the text: one result can be megabytes before truncation. + * + * A repair turn (a wrong-shaped answer; no tool ran) is not a tool turn, so it neither counts nor + * resets: the correction is about the answer's shape and tells the model nothing new about a tool. */ export function toolLoopGuard( agent: string, limit: number, ): (calls: readonly LlmToolCall[], results: readonly LlmToolResult[]) => void { - let last: string | undefined; - let repeats = 0; + const window = MAX_CYCLE * limit; + let recent: readonly { readonly shape: string; readonly tools: readonly string[] }[] = []; return (calls, results) => { const shape = fingerprint({ calls: calls.map((call) => [call.name, call.input]), results: results.map((result) => [result.content, result.isError === true]), }); - repeats = shape === last ? repeats + 1 : 1; - last = shape; - if (repeats >= limit) { - throw new AgentToolLoopError({ agent, repeats, tools: calls.map((call) => call.name) }); + recent = [...recent, { shape, tools: calls.map((call) => call.name) }].slice(-window); + for (let cycle = 1; cycle <= MAX_CYCLE; cycle += 1) { + const laps = recent.slice(-cycle * limit); + if (laps.length < cycle * limit) break; + const repeats = (turn: (typeof laps)[number], index: number): boolean => + index < cycle || isSameTurn(turn.shape, laps[index - cycle]?.shape); + if (!laps.every(repeats)) continue; + const tools = [...new Set(laps.slice(0, cycle).flatMap((turn) => turn.tools))]; + throw new AgentToolLoopError({ agent, repeats: limit, cycle, tools }); } }; } +const isSameTurn = (one: string, other: string | undefined): boolean => one === other; + export interface RunDeadline { /** The caller's signal and the deadline as one — what every provider call is handed. */ readonly signal: AbortSignal; @@ -47,6 +62,12 @@ export interface RunDeadline { * provider's fault. Work that outlives the race is left to finish — a tool has no signal to hear. */ guard(work: Promise): Promise; + /** + * Run `fn` with the deadline on the ambient `ctx.signal`, so a tool — an `action()` reading its + * own `ctx.signal`, a `fetch(url, { signal })` inside it — hears the deadline and can stop. A + * tool that never reads the signal still runs to its own end after the run has been refused. + */ + scope(fn: () => T): T; } /** `ms` undefined is no deadline: the caller's signal, and two no-ops. */ @@ -55,13 +76,16 @@ export function runDeadline( ms: number | undefined, caller: AbortSignal, ): RunDeadline { - if (ms === undefined) return { signal: caller, check: () => {}, guard: (work) => work }; + if (ms === undefined) { + return { signal: caller, check: () => {}, guard: (work) => work, scope: (fn) => fn() }; + } const timer = AbortSignal.timeout(ms); const expired = (): AgentDeadlineError => new AgentDeadlineError({ agent, deadlineMs: ms }); // The caller leaving first stays `X_ABORTED`: it is the truer account of why the run stopped. const byDeadline = (): boolean => timer.aborted && !caller.aborted; return { signal: AbortSignal.any([caller, timer]), + scope: (fn) => withChildContext({ signal: timer }, fn), check: () => { if (byDeadline()) throw expired(); }, diff --git a/packages/ai/src/agent-job.test.ts b/packages/ai/src/agent-job.test.ts index d2bfdacb4..a54617a00 100644 --- a/packages/ai/src/agent-job.test.ts +++ b/packages/ai/src/agent-job.test.ts @@ -9,7 +9,7 @@ import { afterAll, beforeEach, describe, expect, test } from 'bun:test'; import { registerAction, resetActions } from '@ultimat3/action'; -import { ctxOf, userActor } from '@ultimat3/core'; +import { ctxOf, registerErrorRetry, userActor } from '@ultimat3/core'; import type { AnyJobHandle, ClaimedJob, JobDriver, JobExecution } from '@ultimat3/jobs'; import { executeJob, @@ -25,6 +25,7 @@ import { t } from '@ultimat3/schema'; import { agent } from './agent'; import { agentJob } from './agent-job'; import { echoProvider } from './echo-provider'; +import { AI_ERROR_RETRY } from './errors-retry'; import { providerGateway } from './gateway'; import { FIXTURE_MODEL, useFixtureModels } from './model-fixture'; import { definePrompt, type Prompt } from './prompt'; @@ -101,6 +102,7 @@ function summariser( name: string, tools: readonly ProjectableAction[] = [], budget?: { tokensPerRun: number }, + maxRepeatedToolCalls = 100, ) { return agent({ input: Input, @@ -110,7 +112,7 @@ function summariser( tools, ...(budget === undefined ? {} : { budget }), // A scripted model may repeat one call for ever; the ceilings under test are the others. - maxRepeatedToolCalls: 100, + maxRepeatedToolCalls, maxTurns: 6, policy: allow(), }).named(name); @@ -138,6 +140,8 @@ async function runThrough( let driver: JobDriver; beforeEach(() => { + // An app gets this by importing the package; this suite imports its files one by one. + registerErrorRetry(AI_ERROR_RETRY); resetAiRuntime(); resetActions(); resetJobs(); @@ -323,6 +327,62 @@ describe('the ceilings shipped for the request path hold on the job path too', ( expect(provider.seen).toBeLessThan(6); }); + test('a run stopped for repeating itself is dead-lettered on attempt one, never bought three times', async () => { + const provider = scripted(true); + configureAi({ + gateway: providerGateway({ defaultModel: FIXTURE_MODEL, providers: [provider.provider] }), + }); + const handle = agentJob(summariser('loopingJobAgent', [actorProbe([])], undefined, 3), { + name: 'summarise-looping', + tenant: (input) => input.orgId, + retry: { attempts: 3, jitter: false }, + }); + + const execution = await runThrough(handle, driver, { topic: 'a', orgId: 'org-1' }); + expect(execution.error).toContain('X_AGENT_TOOL_LOOP'); + // `attempts: 3` would have re-run the same three turns twice more. + expect(execution.outcome).toBe('dead-lettered'); + expect(provider.seen).toBe(3); + }); + + test('an attempt abandoned mid-call ends as X_ABORTED, and the job policy — not a retryable transport code — decides', async () => { + const aborted = new AbortController(); + let calls = 0; + const hung: Provider = { + name: 'hung', + models: ['claude-opus-5'], + generate(request) { + calls += 1; + setTimeout(() => aborted.abort(), 5); + // A socket that ends only by being aborted. + return new Promise((_, reject) => { + request.signal?.addEventListener('abort', () => reject(request.signal?.reason)); + }); + }, + stream: (request) => echoProvider().stream(request), + }; + configureAi({ gateway: providerGateway({ defaultModel: FIXTURE_MODEL, providers: [hung] }) }); + const handle = agentJob(summariser('abandonedAgent'), { + name: 'summarise-abandoned', + tenant: 'none', + retry: { attempts: 3, jitter: false }, + }); + const execution = await runThrough( + handle, + driver, + { topic: 'a', orgId: 'org-1' }, + ctxOf({ role: 'worker', actor: userActor({ id: 'worker-1' }), signal: aborted.signal }), + ); + // Core's code for "the caller went away" — before, the aborted socket was reported as + // `X_AI_PROVIDER_UNAVAILABLE`, which this package registers `retryable`. + expect(execution.error).toContain('X_ABORTED'); + expect(execution.error).not.toContain('X_AI_PROVIDER_UNAVAILABLE'); + expect(calls).toBe(1); + // `X_ABORTED` is core's and is UNCLASSIFIED for jobs: the attempt count stays in charge. A + // drain or an attempt timeout never reaches this decision — jobs reads those off its own signal. + expect(execution.outcome).toBe('retried'); + }); + test("the job's ctx.signal reaches the agent's turn loop, so a cancelled attempt buys nothing", async () => { const provider = scripted(true); configureAi({ diff --git a/packages/ai/src/agent.ts b/packages/ai/src/agent.ts index 608a77d84..a3adb5a70 100644 --- a/packages/ai/src/agent.ts +++ b/packages/ai/src/agent.ts @@ -319,7 +319,10 @@ async function run< throwIfAborted(args.ctx); deadline.check(); const results: readonly LlmToolResult[] = await deadline.guard( - Promise.all(requested.map((call) => runLlmToolCall(adapted.tools, call, actor))), + // Inside the deadline's scope, so each tool's own `ctx.signal` carries it. + deadline.scope(() => + Promise.all(requested.map((call) => runLlmToolCall(adapted.tools, call, actor))), + ), ); calls += requested.length; // Before the next turn is bought: the same calls with the same results cannot lead diff --git a/packages/ai/src/budget.ts b/packages/ai/src/budget.ts index daf496e5f..796ebcbc3 100644 --- a/packages/ai/src/budget.ts +++ b/packages/ai/src/budget.ts @@ -57,11 +57,12 @@ export interface SpendEstimate { } /** Price a request pre-flight. The pessimistic read on purpose — see `estimateCost`. */ -export function estimateSpend(request: GenerateRequest): SpendEstimate { +export function estimateSpend(request: GenerateRequest, marked = false): SpendEstimate { return { inputTokens: estimateInputTokens(request), tokens: estimateTokens(request), - cost: estimateCost(request), + // `marked`: a provider would send this request with a cache mark, so it can bill as a write. + cost: estimateCost(request, marked), }; } diff --git a/packages/ai/src/errors-retry.test.ts b/packages/ai/src/errors-retry.test.ts index e3620f3a8..8927e8ad0 100644 --- a/packages/ai/src/errors-retry.test.ts +++ b/packages/ai/src/errors-retry.test.ts @@ -3,7 +3,8 @@ import { beforeEach, describe, expect, test } from 'bun:test'; import type { ErrorRetry } from '@ultimat3/core'; -import { declaredErrorRetry, registerErrorRetry } from '@ultimat3/core'; +import { classifyThrown, declaredErrorRetry, registerErrorRetry } from '@ultimat3/core'; +import { AgentDeadlineError, AgentToolLoopError } from './agent-errors'; import { AI_ERROR_CODES, AiBudgetExceededError, @@ -120,18 +121,37 @@ describe('the permanent codes stay terminal', () => { }); describe('the table', () => { - test('one exception, and it is the transport code', () => { - expect(Object.keys(AI_ERROR_RETRY)).toEqual(['X_AI_PROVIDER_UNAVAILABLE']); + test('one retryable code, the transport one; and the two agent stops registered terminal', () => { + expect(Object.keys(AI_ERROR_RETRY)).toEqual([ + 'X_AI_PROVIDER_UNAVAILABLE', + 'X_AGENT_TOOL_LOOP', + 'X_AGENT_DEADLINE', + ]); expect(AI_ERROR_RETRY.X_AI_PROVIDER_UNAVAILABLE).toBe('retryable'); }); - test('nothing is registered AS terminal, and that is the decision, not an oversight', () => { + test('a looped or deadlined run is never re-run by a job: terminal on attempt one', () => { + // Unclassified, a job with `attempts: 3` bought the same repeating run three times. + const stops = [ + new AgentToolLoopError({ agent: 'a@1', repeats: 3, cycle: 1, tools: ['status'] }), + new AgentDeadlineError({ agent: 'a@1', deadlineMs: 30 }), + ]; + for (const error of stops) { + expect(classifyThrown(error), error.code).toBe('terminal'); + expect(rendered(error)).toBe('terminal'); + } + }); + + test('nothing ELSE is registered as terminal, and that is the decision, not an oversight', () => { // `@ultimat3/jobs` dead-letters a REGISTERED `terminal` on attempt 1, where an unclassified // code keeps the job's own attempt count. That is probably right for `X_LLM_REFUSED` and // `X_AI_BUDGET_EXCEEDED` and it is a change to how every app's jobs fail, so it is somebody's // deliberate decision — and this test is what makes adding one a decision. + // The two agent stops are that decision made (2026-10): re-running a run that repeats itself, + // or one that outlived its declared deadline, buys the same run again at full price. + const classified: readonly string[] = Object.keys(AI_ERROR_RETRY); for (const code of AI_ERROR_CODES) { - if (code === 'X_AI_PROVIDER_UNAVAILABLE') continue; + if (classified.includes(code)) continue; expect(atImport.get(code), code).toBeUndefined(); } }); diff --git a/packages/ai/src/errors-retry.ts b/packages/ai/src/errors-retry.ts index 296f0d7b7..8b0f53e93 100644 --- a/packages/ai/src/errors-retry.ts +++ b/packages/ai/src/errors-retry.ts @@ -38,6 +38,12 @@ export const AI_ERROR_RETRY = { // be both, and registering the code here is what makes the override honoured: core reads an // instance `terminal` on an UNREGISTERED code as unclassified. X_AI_PROVIDER_UNAVAILABLE: 'retryable', + // The two stops an unattended `agent()` run makes, REGISTERED terminal — the deliberate decision + // the paragraph above leaves to whoever makes it. A run stopped for repeating itself repeats + // itself again, and a run that outlived its declared deadline is the same run against the same + // deadline: unclassified, an `agentJob` with `attempts: 3` paid for it three times. + X_AGENT_TOOL_LOOP: 'terminal', + X_AGENT_DEADLINE: 'terminal', } as const satisfies Readonly>>; // `Partial`, so the table may be a subset — but every key is still checked against the declared set, diff --git a/packages/ai/src/gateway.ts b/packages/ai/src/gateway.ts index 82689886e..ad47f8690 100644 --- a/packages/ai/src/gateway.ts +++ b/packages/ai/src/gateway.ts @@ -180,7 +180,7 @@ class GatewayImpl implements Gateway { // The estimate is DEBITED here, not merely checked: three concurrent calls under one ledger // all read the same `spent()` otherwise, all pass, and all three record against a ceiling // only one of them fitted. - const reservation = await ledger?.reserve(estimateSpend(resolved)); + const reservation = await ledger?.reserve(estimateSpend(resolved, this.marks(model))); let result: GenerateResult; try { @@ -230,7 +230,7 @@ class GatewayImpl implements Gateway { const provider = this.providerFor(model); const ledger = currentBudget() ?? this.callLedger({}); - const reservation = await ledger?.reserve(estimateSpend(resolved)); + const reservation = await ledger?.reserve(estimateSpend(resolved, this.marks(model))); let settled = false; try { for await (const chunk of provider.stream(resolved)) { @@ -251,6 +251,13 @@ class GatewayImpl implements Gateway { } } + /** Whether any provider serving `model` would send a hinted request with a cache mark. */ + private marks(model: ModelId): boolean { + return this.config.providers.some( + (p) => p.models.includes(model) && p.marksPromptCache?.(model) === true, + ); + } + private providerFor(model: ModelId): Provider { const provider = this.config.providers.find((p) => p.models.includes(model)); if (provider === undefined) { diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 289fd9e3a..2049d89af 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -188,7 +188,13 @@ export type { SystemParts, SystemPrompt, } from './prompt-cache'; -export { PROMPT_CACHE_RULES, promptCacheRule, SESSION_KEY_MAX, systemText } from './prompt-cache'; +export { + OPENAI_CACHE_KEY_MAX, + PROMPT_CACHE_RULES, + promptCacheRule, + SESSION_KEY_MAX, + systemText, +} from './prompt-cache'; export { promptFences } from './prompt-fence'; export type { AiContentBlock, diff --git a/packages/ai/src/openai-body.ts b/packages/ai/src/openai-body.ts index bdbc8d19d..7370cf627 100644 --- a/packages/ai/src/openai-body.ts +++ b/packages/ai/src/openai-body.ts @@ -9,7 +9,7 @@ import { AiRequestInvalidError } from './errors'; import type { Effort, ModelId, ThinkingMode } from './models'; import { modelSpec } from './models'; import { toOpenAiMessages, toOpenAiTools, toolChoiceFor } from './openai-messages'; -import { promptCacheRule, SESSION_KEY_MAX } from './prompt-cache'; +import { OPENAI_CACHE_KEY_MAX, promptCacheRule, SESSION_KEY_MAX } from './prompt-cache'; import type { GenerateRequest } from './provider'; export interface ChatCompletionBodyInput { @@ -56,7 +56,7 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record { + const provider = () => { + register(); + return anthropicProvider({ models: [NATIVE], apiKey: 'k' }); + }; + + test('the stable system block and the last block of the last two messages; three marks', () => { + const body = provider().body({ + model: NATIVE, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 100, + promptCache: { session: 'ignored-here' }, + }); + expect(body['system']).toEqual([ + { type: 'text', text: STABLE, cache_control: MARK }, + { type: 'text', text: RUN }, + ]); + expect(body['messages']).toEqual([ + { role: 'user', content: 'Resolve order 7.' }, + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'call-1', + name: 'lookup', + input: { id: '7' }, + cache_control: MARK, + }, + ], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call-1', content: 'shipped', cache_control: MARK }, + ], + }, + ]); + expect(marks(body)).toBe(3); + }); + + test('a one-message request marks it once; a string message becomes one marked block', () => { + const body = provider().body({ + model: NATIVE, + system: STABLE, + messages: [{ role: 'user', content: 'hello' }], + maxTokens: 100, + promptCache: {}, + }); + expect(body['system']).toEqual([{ type: 'text', text: STABLE, cache_control: MARK }]); + expect(body['messages']).toEqual([ + { role: 'user', content: [{ type: 'text', text: 'hello', cache_control: MARK }] }, + ]); + expect(marks(body)).toBe(2); + }); + + test('no hint: a string system stays a string, two parts are two unmarked blocks', () => { + const plain = provider().body({ + model: NATIVE, + system: STABLE, + messages: transcript, + maxTokens: 9, + }); + expect(plain['system']).toBe(STABLE); + expect(marks(plain)).toBe(0); + const parts = provider().body({ + model: NATIVE, + system: { stable: STABLE, run: RUN }, + messages: transcript, + maxTokens: 9, + }); + expect(parts['system']).toEqual([ + { type: 'text', text: STABLE }, + { type: 'text', text: RUN }, + ]); + expect(marks(parts)).toBe(0); + }); + + test('the mark never lands on an empty or whitespace-only text block: the nearest earlier block takes it', () => { + const body = provider().body({ + model: NATIVE, + system: ' \n ', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'question' }, + { type: 'text', text: '' }, + ], + }, + { role: 'assistant', content: ' ' }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'c', content: 'r' }, + { type: 'text', text: ' \n' }, + ], + }, + ], + maxTokens: 100, + promptCache: {}, + }); + const sent = body['messages'] as { content: unknown }[]; + // The whitespace-only string message is skipped outright; the tail's mark moves back one block. + expect(sent[1]?.content).toBe(' '); + expect(sent[2]?.content).toEqual([ + { type: 'tool_result', tool_use_id: 'c', content: 'r', cache_control: MARK }, + { type: 'text', text: ' \n' }, + ]); + expect(sent[0]?.content).toEqual([ + { type: 'text', text: 'question' }, + { type: 'text', text: '' }, + ]); + // A whitespace-only system is not marked. + expect(JSON.stringify(body['system'])).not.toContain('cache_control'); + expect(marks(body)).toBe(1); + }); + + test('marks a raw caller already placed count against the four: ours are dropped first', () => { + const own = { type: 'text', text: 'pinned', cache_control: MARK } as never; + const withOwn = (count: number): Record => + provider().body({ + model: NATIVE, + system: STABLE, + messages: [ + { role: 'user', content: Array.from({ length: count }, () => own) }, + { role: 'assistant', content: 'thinking' }, + { role: 'user', content: 'go on' }, + ], + maxTokens: 100, + promptCache: {}, + }); + // 1 theirs + 3 ours = 4. 2 theirs: the second-to-last message gives its mark up first. + expect(marks(withOwn(1))).toBe(4); + const two = withOwn(2); + expect(marks(two)).toBe(4); + expect((two['messages'] as { content: unknown }[])[1]?.content).toBe('thinking'); + // 3 theirs: then the system block's; the tail keeps the one that is left. + const three = withOwn(3); + expect(marks(three)).toBe(4); + expect(three['system']).toBe(STABLE); + // 4 theirs: none of ours. + const four = withOwn(4); + expect(marks(four)).toBe(4); + expect((four['messages'] as { content: unknown }[])[2]?.content).toBe('go on'); + }); + + test('an empty stable part is never sent as an empty block, and never marked', () => { + const body = provider().body({ + model: NATIVE, + system: { stable: '', run: RUN }, + messages: [{ role: 'user', content: 'hello' }], + maxTokens: 100, + promptCache: {}, + }); + expect(body['system']).toEqual([{ type: 'text', text: RUN }]); + expect(marks(body)).toBe(1); + }); +}); + +describe('the pre-flight estimate', () => { + test('a hinted request is priced as a cache WRITE, the dearest the prompt can be billed', () => { + register(); + const text = 'x'.repeat(4_000_000); + const request: GenerateRequest = { + model: NATIVE, + messages: [{ role: 'user', content: text }], + maxTokens: 1, + }; + const hinted: GenerateRequest = { ...request, promptCache: {} }; + // 1M tokens at 300 minor; the single output token rounds up to one minor unit. + expect(estimateCost(request).minor).toBe(301); + // A hint alone changes nothing: the price moves only when a provider WOULD mark this request. + expect(estimateCost(hinted).minor).toBe(301); + expect(estimateCost(request, true).minor).toBe(301); + // …and then it is 1.25x. + expect(estimateCost(hinted, true).minor).toBe(376); + }); + + test('a row whose write is cheaper than input is still priced at the input rate', () => { + register(); + registerModel({ id: NATIVE, ...ROW, cacheWritePerMillion: usd(0) }); + const request: GenerateRequest = { + model: NATIVE, + messages: [{ role: 'user', content: 'x'.repeat(4_000_000) }], + maxTokens: 1, + promptCache: {}, + }; + // Priced at the write rate alone this was 1: a `costPerCall` ceiling that passes everything. + expect(estimateCost(request, true).minor).toBe(301); + }); + + test('the gateway prices the write only where a provider would mark: not Gemini, not an unknown host', async () => { + register(); + const { fetch } = capture(CHAT_REPLY); + const text = 'x'.repeat(4_000_000); + const reserve = async (model: string, baseUrl: string): Promise => { + const provider = openAiProvider({ apiKey: 'k', baseUrl, models: [CLAUDE, GEMINI], fetch }); + const gateway = providerGateway({ providers: [provider] }); + const ledger = new BudgetLedger({ limits: {} }); + const reserved: number[] = []; + const original = ledger.reserve.bind(ledger); + ledger.reserve = (estimate) => { + reserved.push(estimate.cost.minor); + return original(estimate); + }; + await withBudget(ledger, () => + gateway.generate({ + model, + messages: [{ role: 'user', content: text }], + maxTokens: 1, + promptCache: {}, + }), + ); + return reserved[0] ?? expect.unreachable('nothing was reserved'); + }; + expect(await reserve(CLAUDE, 'https://openrouter.ai/api/v1')).toBe(376); + expect(await reserve(GEMINI, 'https://openrouter.ai/api/v1')).toBe(301); + expect(await reserve(CLAUDE, 'https://llm.acme.internal/v1')).toBe(301); + }); +}); diff --git a/packages/ai/src/prompt-cache-blocks.ts b/packages/ai/src/prompt-cache-blocks.ts index 88c66fd1d..a10d579d4 100644 --- a/packages/ai/src/prompt-cache-blocks.ts +++ b/packages/ai/src/prompt-cache-blocks.ts @@ -4,21 +4,65 @@ import type { SystemPrompt } from './prompt-cache'; import { CACHE_MARK, systemParts } from './prompt-cache'; -import type { AiMessage } from './provider'; +import type { AiMessage, GenerateRequest } from './provider'; type Marked = T & { readonly cache_control?: typeof CACHE_MARK }; type TextBlock = Marked<{ readonly type: 'text'; readonly text: string }>; +type WireMessage = { readonly role: AiMessage['role']; readonly content: unknown }; -/** How many trailing messages carry a mark. With the system block: three of the four breakpoints. */ -const MARKED_TAIL = 2; +/** The vendor's ceiling on breakpoints in one request; a fifth is a 400. */ +const MAX_BREAKPOINTS = 4; + +/** An empty or whitespace-only text block cannot carry a mark (and is a 400 with one). */ +const blank = (text: string): boolean => text.trim() === ''; + +export interface AnthropicPrompt { + readonly system: string | readonly TextBlock[] | undefined; + readonly messages: readonly WireMessage[]; +} /** - * `system`, as the wire takes it. A plain string with no mark stays the string every existing - * request sent. Otherwise blocks, the stable one FIRST and the only one marked: it is the half - * other runs share, and the per-run half after it is covered by the transcript's own marks. - * An empty part is dropped — an empty text block is a 400, and a mark on one is too. + * `system` and `messages` as the wire takes them. Unmarked, a string system stays the string every + * existing request sent and the messages pass through untouched. + * + * Marked, at most three breakpoints, spent in this order: the transcript's TAIL (what the next + * request reads back), the stable system block (what other runs share), then the message before + * the tail (a second start for the 20-block lookback). Marks the caller already placed — a raw + * block or tool carrying its own `cache_control` — count against the four first, so this never + * turns a legal request into a 400; ours are the ones dropped. */ -export function anthropicSystem( +export function anthropicPrompt(request: GenerateRequest, marked: boolean): AnthropicPrompt { + const { messages } = request; + if (!marked) { + return { system: systemOf(request.system, false), messages: messages.map(plain) }; + } + let left = MAX_BREAKPOINTS - placedBy(request); + const take = (wanted: boolean): boolean => { + if (!wanted || left <= 0) return false; + left -= 1; + return true; + }; + const tail = messages.length - 1; + const markTail = take(tail >= 0 && markableAt(messages[tail]) !== undefined); + const markSystem = take(!blank(systemParts(request.system).stable)); + const markPrevious = take(tail >= 1 && markableAt(messages[tail - 1]) !== undefined); + return { + system: systemOf(request.system, markSystem), + messages: messages.map((message, index) => + (index === tail && markTail) || (index === tail - 1 && markPrevious) + ? { role: message.role, content: markedContent(message.content) } + : plain(message), + ), + }; +} + +const plain = (message: AiMessage): WireMessage => ({ + role: message.role, + content: message.content, +}); + +/** The stable block first and the only one marked; an empty part is dropped, never sent. */ +function systemOf( system: SystemPrompt | undefined, marked: boolean, ): string | readonly TextBlock[] | undefined { @@ -34,28 +78,37 @@ export function anthropicSystem( } /** - * The transcript, with the last block of the last two messages marked. The tail is what the NEXT - * request reads back; the one before it gives the lookup a second start, so a turn that appended - * more than the 20 blocks one breakpoint looks back over still finds the previous turn's entry. + * Where one message's mark goes: its LAST block that is not blank text, so the mark moves back + * past a trailing empty block rather than landing on it. `undefined` when no block can carry one. */ -export function anthropicMessages( - messages: readonly AiMessage[], - marked: boolean, -): readonly { readonly role: AiMessage['role']; readonly content: unknown }[] { - const from = marked ? messages.length - MARKED_TAIL : messages.length; - return messages.map((message, index) => ({ - role: message.role, - content: index >= from ? markedContent(message.content) : message.content, - })); +function markableAt(message: AiMessage | undefined): number | undefined { + if (message === undefined) return undefined; + const { content } = message; + if (typeof content === 'string') return blank(content) ? undefined : 0; + for (let index = content.length - 1; index >= 0; index -= 1) { + const block = content[index]; + if (block !== undefined && !(block.type === 'text' && blank(block.text))) return index; + } + return undefined; } function markedContent(content: AiMessage['content']): unknown { if (typeof content === 'string') { - // An empty text block cannot be marked (or sent); the string goes as it stood. - return content === '' ? content : [{ type: 'text', text: content, cache_control: CACHE_MARK }]; + return [{ type: 'text', text: content, cache_control: CACHE_MARK }]; } - const last = content.length - 1; + const at = markableAt({ role: 'user', content }); return content.map((block, index) => - index === last ? { ...block, cache_control: CACHE_MARK } : block, + index === at ? { ...block, cache_control: CACHE_MARK } : block, ); } + +/** Breakpoints the caller placed itself: the types do not offer the field, a cast still sends it. */ +function placedBy(request: GenerateRequest): number { + const carries = (value: unknown): boolean => + typeof value === 'object' && value !== null && Object.hasOwn(value, 'cache_control'); + let count = (request.tools ?? []).filter(carries).length; + for (const { content } of request.messages) { + if (typeof content !== 'string') count += content.filter(carries).length; + } + return count; +} diff --git a/packages/ai/src/prompt-cache-fixture.ts b/packages/ai/src/prompt-cache-fixture.ts new file mode 100644 index 000000000..c446566a8 --- /dev/null +++ b/packages/ai/src/prompt-cache-fixture.ts @@ -0,0 +1,86 @@ +// What the two prompt-cache wire suites share: the rows they register, one transcript, and a fetch +// that records what the socket was handed. Not shipped (`!src/**/*-fixture.ts`). + +import { expect } from 'bun:test'; +import { registerModel } from './models'; +import { openAiProvider } from './openai-provider'; +import type { GenerateRequest } from './provider'; + +export const usd = (minor: number) => ({ minor, currency: 'USD' }); +export const ROW = { + contextWindow: 200_000, + maxOutput: 8_000, + inputPerMillion: usd(300), + outputPerMillion: usd(1_500), + cacheMinimumTokens: 1_024, + reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, +} as const; +export const CLAUDE = 'anthropic/claude-sonnet-5.5'; +export const GEMINI = 'google/gemini-2.5-flash'; +export const GPT = 'gpt-5.6-luna'; +export const NATIVE = 'claude-sonnet-5-5'; + +export function register(): void { + for (const id of [CLAUDE, GEMINI, GPT, NATIVE]) registerModel({ id, ...ROW }); +} + +export const MARK = { type: 'ephemeral' }; +export const STABLE = 'You are the firm’s assistant. RULES…'; +export const RUN = 'Today is 2026-10-11.'; + +/** A three-message transcript: the shape turn 2 of a tool loop sends. */ +export const transcript: GenerateRequest['messages'] = [ + { role: 'user', content: 'Resolve order 7.' }, + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'call-1', name: 'lookup', input: { id: '7' } }], + }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call-1', content: 'shipped' }] }, +]; + +export interface Sent { + readonly url: string; + readonly headers: Record; + readonly body: Record; +} + +export function capture(reply: unknown): { sent: Sent[]; fetch: typeof fetch } { + const sent: Sent[] = []; + const fake = (async (url: string | URL | Request, init?: RequestInit) => { + sent.push({ + url: String(url), + headers: { ...(init?.headers as Record) }, + body: JSON.parse(String(init?.body)) as Record, + }); + return new Response(JSON.stringify(reply), { status: 200 }); + }) as typeof fetch; + return { sent, fetch: fake }; +} + +export const CHAT_REPLY = { + choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 1 }, +}; + +export async function openRouter( + request: GenerateRequest, + baseUrl = 'https://openrouter.ai/api/v1', +) { + register(); + const { sent, fetch } = capture(CHAT_REPLY); + const provider = openAiProvider({ apiKey: 'k', baseUrl, models: [CLAUDE, GEMINI, GPT], fetch }); + await provider.generate(request); + const only = sent[0]; + if (only === undefined) return expect.unreachable('no request was sent'); + return only; +} + +/** Every `cache_control` anywhere in a body — the count a vendor caps at four. */ +export function marks(value: unknown): number { + if (Array.isArray(value)) return value.reduce((sum: number, item) => sum + marks(item), 0); + if (typeof value !== 'object' || value === null) return 0; + return Object.entries(value).reduce( + (sum, [key, child]) => sum + (key === 'cache_control' ? 1 : marks(child)), + 0, + ); +} diff --git a/packages/ai/src/prompt-cache.test.ts b/packages/ai/src/prompt-cache.test.ts index 0fac7e031..195f91c81 100644 --- a/packages/ai/src/prompt-cache.test.ts +++ b/packages/ai/src/prompt-cache.test.ts @@ -1,90 +1,28 @@ /** * What LEAVES the process when a request carries a prompt-cache hint — one golden request per rule - * in `PROMPT_CACHE_RULES`, asserted on the body and headers the socket is handed. + * in `PROMPT_CACHE_RULES` on the chat-completions wire, asserted on the body and headers the socket + * is handed. The Anthropic wire and the pre-flight estimate are `prompt-cache-blocks.test.ts`. */ import { describe, expect, test } from 'bun:test'; -import { registerModel } from './models'; import { openAiProvider } from './openai-provider'; -import { promptCacheRule, SESSION_KEY_MAX, systemText } from './prompt-cache'; +import { OPENAI_CACHE_KEY_MAX, promptCacheRule, SESSION_KEY_MAX, systemText } from './prompt-cache'; +import { + CHAT_REPLY, + CLAUDE, + capture, + GEMINI, + GPT, + MARK, + marks, + NATIVE, + openRouter, + RUN, + register, + STABLE, + transcript, +} from './prompt-cache-fixture'; import type { GenerateRequest } from './provider'; -import { anthropicProvider, estimateCost } from './provider'; - -const usd = (minor: number) => ({ minor, currency: 'USD' }); -const ROW = { - contextWindow: 200_000, - maxOutput: 8_000, - inputPerMillion: usd(300), - outputPerMillion: usd(1_500), - cacheMinimumTokens: 1_024, - reasoning: { effort: false, adaptive: false, disableThinkingUpTo: undefined }, -} as const; -const CLAUDE = 'anthropic/claude-sonnet-5.5'; -const GEMINI = 'google/gemini-2.5-flash'; -const GPT = 'gpt-5.6-luna'; -const NATIVE = 'claude-sonnet-5-5'; - -function register(): void { - for (const id of [CLAUDE, GEMINI, GPT, NATIVE]) registerModel({ id, ...ROW }); -} - -const MARK = { type: 'ephemeral' }; -const STABLE = 'You are the firm’s assistant. RULES…'; -const RUN = 'Today is 2026-10-11.'; - -/** A three-message transcript: the shape turn 2 of a tool loop sends. */ -const transcript: GenerateRequest['messages'] = [ - { role: 'user', content: 'Resolve order 7.' }, - { - role: 'assistant', - content: [{ type: 'tool_use', id: 'call-1', name: 'lookup', input: { id: '7' } }], - }, - { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call-1', content: 'shipped' }] }, -]; - -interface Sent { - readonly url: string; - readonly headers: Record; - readonly body: Record; -} - -function capture(reply: unknown): { sent: Sent[]; fetch: typeof fetch } { - const sent: Sent[] = []; - const fake = (async (url: string | URL | Request, init?: RequestInit) => { - sent.push({ - url: String(url), - headers: { ...(init?.headers as Record) }, - body: JSON.parse(String(init?.body)) as Record, - }); - return new Response(JSON.stringify(reply), { status: 200 }); - }) as typeof fetch; - return { sent, fetch: fake }; -} - -const CHAT_REPLY = { - choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], - usage: { prompt_tokens: 10, completion_tokens: 1 }, -}; - -async function openRouter(request: GenerateRequest, baseUrl = 'https://openrouter.ai/api/v1') { - register(); - const { sent, fetch } = capture(CHAT_REPLY); - const provider = openAiProvider({ apiKey: 'k', baseUrl, models: [CLAUDE, GEMINI, GPT], fetch }); - await provider.generate(request); - const only = sent[0]; - if (only === undefined) return expect.unreachable('no request was sent'); - return only; -} - -/** Every `cache_control` anywhere in a body — the count a vendor caps at four. */ -function marks(value: unknown): number { - if (Array.isArray(value)) return value.reduce((sum: number, item) => sum + marks(item), 0); - if (typeof value !== 'object' || value === null) return 0; - return Object.entries(value).reduce( - (sum, [key, child]) => sum + (key === 'cache_control' ? 1 : marks(child)), - 0, - ); -} describe('the rule table', () => { test('first match wins, per wire, host and model prefix', () => { @@ -264,6 +202,24 @@ describe('OpenAI direct', () => { expect(bare.body['prompt_cache_key']).toBeUndefined(); }); + test('a key over 64 characters is hashed for prompt_cache_key — the same key, the same hash', async () => { + const request: GenerateRequest = { model: GPT, messages: transcript, maxTokens: 100 }; + const long = `tenant:${'a'.repeat(100)}`; + const send = async (session: string) => + (await openRouter({ ...request, promptCache: { session } }, 'https://api.openai.com/v1')) + .body['prompt_cache_key'] as string; + const hashed = await send(long); + expect(hashed).toMatch(/^h:[0-9a-f]{62}$/); + expect(hashed).toHaveLength(OPENAI_CACHE_KEY_MAX); + expect(await send(long)).toBe(hashed); + expect(await send(`${long}b`)).not.toBe(hashed); + // At the limit, verbatim. + expect(await send('k'.repeat(OPENAI_CACHE_KEY_MAX))).toBe('k'.repeat(OPENAI_CACHE_KEY_MAX)); + // The same 100+ characters are a legal OpenRouter session id, and travel as written. + const routed = await openRouter({ ...request, model: GEMINI, promptCache: { session: long } }); + expect(routed.headers['x-session-id']).toBe(long); + }); + test('an endpoint the table does not name gets neither mark nor session', async () => { const { body, headers } = await openRouter( { @@ -280,112 +236,3 @@ describe('OpenAI direct', () => { expect(body['prompt_cache_key']).toBeUndefined(); }); }); - -describe('Anthropic Messages wire', () => { - const provider = () => { - register(); - return anthropicProvider({ models: [NATIVE], apiKey: 'k' }); - }; - - test('the stable system block and the last block of the last two messages; three marks', () => { - const body = provider().body({ - model: NATIVE, - system: { stable: STABLE, run: RUN }, - messages: transcript, - maxTokens: 100, - promptCache: { session: 'ignored-here' }, - }); - expect(body['system']).toEqual([ - { type: 'text', text: STABLE, cache_control: MARK }, - { type: 'text', text: RUN }, - ]); - expect(body['messages']).toEqual([ - { role: 'user', content: 'Resolve order 7.' }, - { - role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'call-1', - name: 'lookup', - input: { id: '7' }, - cache_control: MARK, - }, - ], - }, - { - role: 'user', - content: [ - { type: 'tool_result', tool_use_id: 'call-1', content: 'shipped', cache_control: MARK }, - ], - }, - ]); - expect(marks(body)).toBe(3); - }); - - test('a one-message request marks it once; a string message becomes one marked block', () => { - const body = provider().body({ - model: NATIVE, - system: STABLE, - messages: [{ role: 'user', content: 'hello' }], - maxTokens: 100, - promptCache: {}, - }); - expect(body['system']).toEqual([{ type: 'text', text: STABLE, cache_control: MARK }]); - expect(body['messages']).toEqual([ - { role: 'user', content: [{ type: 'text', text: 'hello', cache_control: MARK }] }, - ]); - expect(marks(body)).toBe(2); - }); - - test('no hint: a string system stays a string, two parts are two unmarked blocks', () => { - const plain = provider().body({ - model: NATIVE, - system: STABLE, - messages: transcript, - maxTokens: 9, - }); - expect(plain['system']).toBe(STABLE); - expect(marks(plain)).toBe(0); - const parts = provider().body({ - model: NATIVE, - system: { stable: STABLE, run: RUN }, - messages: transcript, - maxTokens: 9, - }); - expect(parts['system']).toEqual([ - { type: 'text', text: STABLE }, - { type: 'text', text: RUN }, - ]); - expect(marks(parts)).toBe(0); - }); - - test('an empty stable part is never sent as an empty block, and never marked', () => { - const body = provider().body({ - model: NATIVE, - system: { stable: '', run: RUN }, - messages: [{ role: 'user', content: 'hello' }], - maxTokens: 100, - promptCache: {}, - }); - expect(body['system']).toEqual([{ type: 'text', text: RUN }]); - expect(marks(body)).toBe(1); - }); -}); - -describe('the pre-flight estimate', () => { - test('a hinted request is priced as a cache WRITE, the dearest the prompt can be billed', () => { - register(); - const text = 'x'.repeat(4_000_000); - const request: GenerateRequest = { - model: NATIVE, - messages: [{ role: 'user', content: text }], - maxTokens: 1, - }; - const plain = estimateCost(request).minor; - const hinted = estimateCost({ ...request, promptCache: {} }).minor; - // 1M tokens at 300 minor, then 1.25x; the single output token rounds up to one minor unit. - expect(plain).toBe(301); - expect(hinted).toBe(376); - }); -}); diff --git a/packages/ai/src/prompt-cache.ts b/packages/ai/src/prompt-cache.ts index 921c5fd41..15e54665d 100644 --- a/packages/ai/src/prompt-cache.ts +++ b/packages/ai/src/prompt-cache.ts @@ -39,6 +39,14 @@ // read turn 1's 19,914-token prefix and wrote 72), and `prompt_tokens` INCLUDES both the read and // the written tokens — which is why ./openai-wire subtracts both. // +// Two limits of the rows below, stated rather than hidden: +// - the `anthropic` row matches ANY host an `anthropicProvider({ baseUrl })` is pointed at — a +// proxy speaking that wire is assumed to speak its `cache_control` too; +// - the `parts` row marks the tail only, so the next turn finds this turn's entry through the +// 20-position lookback alone. A turn that appends more blocks than that — more than about 9 +// tool calls in ONE turn, each a `tool_use` and a `tool_result` — misses it and is billed as a +// fresh write of the whole transcript. The stable system block still reads. +// // The table is DATA: a new host or model family is a row. Unknown endpoints get NOTHING — a field // a self-hosted server has never heard of is a 400 on some of them, and a mark is never worth that. @@ -70,9 +78,17 @@ export interface PromptCacheHint { readonly session?: string; } -/** OpenRouter's documented ceiling on a session id; OpenAI's key is held to the same one. */ +/** OpenRouter's documented ceiling on a session id ("Maximum length is 256 characters"). */ export const SESSION_KEY_MAX = 256; +/** + * The longest `prompt_cache_key` sent to OpenAI verbatim. OpenAI's prompt-caching guide and its + * SDK types (read 2026-10-11) state NO maximum, and its API reference could not be read from + * here; 64 is the conservative figure a reviewer recalled. A longer key is not refused — it is a + * grouping key, so it is replaced by a deterministic hash of itself: same key, same group. + */ +export const OPENAI_CACHE_KEY_MAX = 64; + /** The one mark this package sends: the 5-minute TTL, which is the rate `costOf` prices. */ export const CACHE_MARK = { type: 'ephemeral' } as const; diff --git a/packages/ai/src/provider.ts b/packages/ai/src/provider.ts index be25f3b5a..7a6ce2889 100644 --- a/packages/ai/src/provider.ts +++ b/packages/ai/src/provider.ts @@ -14,7 +14,7 @@ import type { Effort, ModelId, ThinkingMode } from './models'; import { modelSpec, reasoningBody } from './models'; import type { PromptCacheHint, SystemPrompt } from './prompt-cache'; import { promptCacheRule, systemText } from './prompt-cache'; -import { anthropicMessages, anthropicSystem } from './prompt-cache-blocks'; +import { anthropicPrompt } from './prompt-cache-blocks'; import { readSse } from './sse'; import type { LlmTool, LlmToolCall } from './tools'; import { @@ -164,6 +164,12 @@ export interface Provider { readonly models: readonly ModelId[]; generate(request: GenerateRequest): Promise; stream(request: GenerateRequest): AsyncIterable; + /** + * Whether a request carrying `promptCache` for this model would be sent with an explicit cache + * mark — billed as a cache WRITE. Absent is no: the gateway prices the write only where one + * can happen. + */ + marksPromptCache?(model: ModelId): boolean; } /** @@ -333,22 +339,26 @@ export class AnthropicProvider implements Provider { }; } + marksPromptCache(model: ModelId): boolean { + return promptCacheRule('anthropic', this.config.baseUrl, model).marks === 'blocks'; + } + /** The request body. Pure and side-effect free so a test can assert it directly. */ body(request: GenerateRequest): Record { const model = this.modelOf(request); // The blocks pass through untouched below — they ARE this wire's shapes — so this is the screen. assertMediaContent(request.messages, { provider: this.name, model }); - const marked = - request.promptCache !== undefined && - promptCacheRule('anthropic', this.config.baseUrl, model).marks === 'blocks'; + const prompt = anthropicPrompt( + request, + request.promptCache !== undefined && this.marksPromptCache(model), + ); const body: Record = { model, max_tokens: Math.min(request.maxTokens, modelSpec(model).maxOutput), - messages: anthropicMessages(request.messages, marked), + messages: prompt.messages, ...reasoningBody(model, request.effort, request.thinking), }; - const system = anthropicSystem(request.system, marked); - if (system !== undefined) body['system'] = system; + if (prompt.system !== undefined) body['system'] = prompt.system; if (request.tools !== undefined && request.tools.length > 0) body['tools'] = request.tools; if (request.stopSequences !== undefined) body['stop_sequences'] = request.stopSequences; return body; @@ -462,17 +472,17 @@ export function estimateInputTokens(request: GenerateRequest): number { * `maxTokens` at the output rate. Deliberately pessimistic — a ceiling checked against an * optimistic estimate is a ceiling one long completion walks through. */ -export function estimateCost(request: GenerateRequest): Money { - // A hinted request can be billed as a cache WRITE end to end (turn one of a run), which is dearer - // than input — so that is the rate the ceiling is checked against. +export function estimateCost(request: GenerateRequest, marked = false): Money { + const model = resolveModel('provider', request.model); const prompt = estimateInputTokens(request); - const write = request.promptCache !== undefined; - return costOf(resolveModel('provider', request.model), { - inputTokens: write ? 0 : prompt, - outputTokens: completionCeiling(request), - cacheReadTokens: 0, - cacheWriteTokens: write ? prompt : 0, - }); + const usage = { outputTokens: completionCeiling(request), cacheReadTokens: 0 }; + const asInput = costOf(model, { ...usage, inputTokens: prompt, cacheWriteTokens: 0 }); + // A request a provider WILL mark (`marked`: the gateway asks the providers serving the model) can + // be billed as a cache write end to end. The DEARER of the two: a row whose write is cheaper + // than input must not lower the ceiling's estimate below what an unmarked call costs. + if (!marked || request.promptCache === undefined) return asInput; + const asWrite = costOf(model, { ...usage, inputTokens: 0, cacheWriteTokens: prompt }); + return asWrite.minor > asInput.minor ? asWrite : asInput; } export function estimateTextTokens(text: string): number { diff --git a/wiki/Agents.md b/wiki/Agents.md index e202d8448..b67c5fd0b 100644 --- a/wiki/Agents.md +++ b/wiki/Agents.md @@ -337,14 +337,19 @@ export const kbAgent = agent({ |---|---| | The stable block must be the same **bytes** | a cache is a prefix match. A timestamp, a request id or an unsorted list in `stable` is a full-price write on every call — put it in `run` | | Below the model's minimum nothing is cached | 512 to 4,096 tokens depending on the model (`ModelSpec.cacheMinimumTokens`); a shorter prefix is processed uncached, with no error | -| `session` is a routing key, not a secret | 1 to 256 printable ASCII characters, else `X_AI_REQUEST_INVALID`. It leaves the process in a header | +| `session` is a routing key, not a secret | on OpenRouter it leaves the process in a header: 1 to 256 printable ASCII characters, else `X_AI_REQUEST_INVALID`. On OpenAI a key over 64 characters is sent as a hash of itself (`h:` + 62 hex) — the same key always groups the same requests. OpenAI's guide states no maximum; 64 is a conservative ceiling `As of 2026-10` | +| The Anthropic row matches any host | `anthropicProvider({ baseUrl })` pointed at a proxy still sends `cache_control`: a server speaking that wire is assumed to speak its marks | +| Never an empty block, never a fifth mark | a mark moves back past an empty or whitespace-only text block, or is not sent; marks a raw request already carries count against the vendor's four, and the framework's are dropped first — the message before the tail, then the system block, then the tail | +| More than about 9 tool calls in **one** turn, on OpenRouter | that turn's transcript is not read back: only the tail is marked there, the provider looks back 20 blocks from it, and each call adds two. The next turn pays a write for the transcript; the stable system block still reads | | No routing preference is ever sent | OpenRouter skips sticky routing when a request orders its providers, and a cache lives on one upstream | | `llm()` is **off** by default | `promptCache: 'auto'` on an `llm()` that is called many times with one long `system`. A one-shot call that marks its prefix pays the write premium for a cache nothing reads | | A direct `gateway.generate()` call | `system: { stable, run }` and `promptCache: {}` on the request — the same two fields the loop sets | **Reading it back.** `onTurn`'s `usage` carries `cacheReadTokens` and `cacheWriteTokens` per turn, `run` carries the running total (`run.usage`, `run.cost`), and the `agent.turn` span event carries `llm.cache.read_tokens` / `llm.cache.write_tokens`. A run whose later turns read nothing has an unstable prefix. -**What it costs.** `costOf` prices a read at the model row's `cacheReadPerMillion` (absent: 0.1x input) and a write at `cacheWritePerMillion` (absent: 1.25x input). A token budget (`tokensPerRun`, `tokensIn`, the gateway's actor and org ceilings) counts **every** token the provider processed — input, output, cache read and cache write, each as one — so caching lowers what a run costs, not how many tokens it is allowed. The pre-flight estimate of a marked request prices the whole prompt as a cache write, the dearest it can be billed, so `costPerCall` holds on turn one. +**What it costs.** `costOf` prices a read at the model row's `cacheReadPerMillion` (absent: 0.1x input) and a write at `cacheWritePerMillion` (absent: 1.25x input). A token budget (`tokensPerRun`, `tokensIn`, the gateway's actor and org ceilings) counts **every** token the provider processed — input, output, cache read and cache write, each as one — so caching lowers what a run costs, not how many tokens it is allowed. The pre-flight estimate prices the prompt as a cache write **only when a provider serving the model would actually mark the request** (an Anthropic model; not Gemini, OpenAI or an unknown host, whose estimate is what it was), and at the dearer of the write and input rates — so `costPerCall` holds on turn one. + +**Two totals, and which to read.** `run.cost` is the run's usage summed and priced once. The ledger — `gateway.spent()`, and what `costPerCall` and every token ceiling are enforced against — adds each turn's cost, and each turn's cost is rounded **up** to a minor unit on its input half and again on its output half. So the ledger can exceed `run.cost` by under 2 minor units per turn. Enforce on the ledger (declare a `budget`); report `run.cost`. Measured, 2026-10-11, one three-turn run against OpenRouter with a 19,914-token stable prefix: turn 1 wrote 19,914 tokens and read 0; turn 2 read 19,914 and wrote 72; turn 3 read 19,986 and wrote 68. @@ -354,11 +359,13 @@ Three bounds stop a run nobody is watching, beside `maxTurns` and `budget.tokens | Bound | Default | Reaching it | |---|---|---| -| `maxRepeatedToolCalls` | **3** | `X_AGENT_TOOL_LOOP` — that many consecutive turns made the **same tool calls and read the same results**. A polled tool whose answer changes is progress and is left alone | -| `deadlineMs` | none | `X_AGENT_DEADLINE` — the run was still going; the model call in flight is aborted on the socket, a tool still running is no longer waited for, and no further turn starts | +| `maxRepeatedToolCalls` | **3** | `X_AGENT_TOOL_LOOP` — the most recent tool turns are one cycle gone round that many times: the same turn over and over, or a cycle of 2 or 3 turns (A,B,A,B…). "The same" is the calls, their arguments, the results and whether each failed, so a polled tool whose answer changes is progress and is left alone. A repair turn (a wrong-shaped answer) is not a tool turn: it neither counts nor resets. A longer cycle is `maxTurns`' to stop | +| `deadlineMs` | none | `X_AGENT_DEADLINE` — the run was still going; the model call in flight is aborted on the socket, every tool's own `ctx.signal` fires so a tool that reads it stops, and no further turn starts. A tool that ignores its signal is no longer waited for but **does run to its own end** — its effects can land after the run was refused | | `maxToolResultChars` | **4,000** | not an error — the result is truncated, and says so in the transcript | -A caller that goes away mid-call is `X_ABORTED`, not the transport failure the aborted socket would otherwise report — that one is retryable, and a job would re-run a run nobody is waiting for. +**Under a job.** `X_AGENT_TOOL_LOOP` and `X_AGENT_DEADLINE` are registered `terminal`: an `agentJob()` dead-letters on the first attempt instead of buying the same run once per `retry.attempts`. + +A caller that goes away mid-call is `X_ABORTED`, not the transport failure (`X_AI_PROVIDER_UNAVAILABLE`, retryable) the aborted socket used to be reported as. `X_ABORTED` is core's and unclassified, so over HTTP it reads `terminal`; in a job the attempt count stays in charge, and a drain or an attempt timeout is settled by jobs off its own signal as before. ## Progress (`onTurn`) diff --git a/wiki/Error-Codes.md b/wiki/Error-Codes.md index e0c0ac3a1..6d7eaa5ee 100644 --- a/wiki/Error-Codes.md +++ b/wiki/Error-Codes.md @@ -707,8 +707,8 @@ Three of them are raised at `notifier(...)`, while the app's modules load, so th | `X_MCP_CONFIRMATION_CONTESTED` | identical concurrent MCP calls kept racing one confirmation | one agent sent the same call concurrently and the open row kept being consumed | `tools/call` — retry once, after the concurrent identical calls finish: send one at a time | | `X_MCP_CONFIRMATION_ARGUMENTS_MISMATCH` | an MCP confirmation approval did not carry the arguments the agent sent | approve was sent without arguments, with different arguments than view returned, or on a row keyed before a signing-secret rotation | `confirmations({ id, decision: 'view' })` — then approve with exactly the arguments it returns | | `X_AI_MODEL_UNRESOLVED` | a model call named no model | neither the declaration, its prompt, nor providerGateway({ defaultModel }) named one — the framework chooses no model for an app | `llm({ ..., model: modelId })` — or model: modelId in its definePrompt, or providerGateway({ ..., defaultModel: modelId }); register the id with registerModel | -| `X_AGENT_TOOL_LOOP` | an agent repeated one tool call with one result and was stopped | an agent() asked for the same tool calls with the same arguments and read the same results on consecutive turns, so no later turn can differ. Stopped at maxRepeatedToolCalls (default 3) instead of spending every remaining turn re-sending a growing transcript for the same non-answer | `maxRepeatedToolCalls: 5` — on the agent, only for a tool that is polled and answers identically while it waits; otherwise tell the template what to do when a tool result does not change, then bump its version | -| `X_AGENT_DEADLINE` | an agent ran past its declared deadline and was stopped | an agent() declared `deadlineMs` and the run was still going when it elapsed: the model call in flight was aborted and no further turn or tool was started. The run has no answer; tools that already ran keep their effects | `deadlineMs: 300_000` — on the agent, only once a trace shows the run converging; otherwise lower `maxTurns`, or shorten the slow tool the `agent.turn` span events name | +| `X_AGENT_TOOL_LOOP` | an agent repeated one tool call with one result and was stopped | an agent() asked for the same tool calls with the same arguments and read the same results on consecutive turns — or went round a cycle of 2 or 3 such turns — so no later turn can differ. Stopped at maxRepeatedToolCalls (default 3) laps instead of spending every remaining turn re-sending a growing transcript for the same non-answer. Terminal: a job does not retry it | `maxRepeatedToolCalls: 5` — on the agent, only for a tool that is polled and answers identically while it waits; otherwise tell the template what to do when a tool result does not change, then bump its version | +| `X_AGENT_DEADLINE` | an agent ran past its declared deadline and was stopped | an agent() declared `deadlineMs` and the run was still going when it elapsed: the model call in flight was aborted, each running tool's `ctx.signal` fired, and no further turn or tool was started. The run has no answer; tools that already ran keep their effects, and one that ignores its signal still finishes. Terminal: a job does not retry it | `deadlineMs: 300_000` — on the agent, only once a trace shows the run converging; otherwise lower `maxTurns`, or shorten the slow tool the `agent.turn` span events name | ## Scraping From 9ffbcbf1af2d6a70bc3cc0bae281a8f6a4e80c40 Mon Sep 17 00:00:00 2001 From: sebi Date: Sun, 11 Oct 2026 09:52:29 -0500 Subject: [PATCH 3/3] fix(ai): the deadline error's cause is true for every phase it guards Co-Authored-By: Claude Opus 5.5 --- packages/ai/src/agent-errors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/src/agent-errors.ts b/packages/ai/src/agent-errors.ts index 1e75bf50e..d0baa4e76 100644 --- a/packages/ai/src/agent-errors.ts +++ b/packages/ai/src/agent-errors.ts @@ -42,7 +42,7 @@ export class AgentDeadlineError extends UltimateError { constructor(input: { agent: string; deadlineMs: number }) { super({ code: 'X_AGENT_DEADLINE', - cause: `agent "${input.agent}" was still running after its ${input.deadlineMs}ms deadline; the model call in flight was aborted and no further turn or tool was started`, + cause: `agent "${input.agent}" was still running after its ${input.deadlineMs}ms deadline; whatever was in flight (setup, a model call or a tool) was signalled to stop, and no further turn or tool was started`, fix: 'deadlineMs: 300_000 # on the agent, only once a trace shows the run converging — otherwise lower maxTurns, or shorten the slow tool the agent.turn span events name', meta: { agent: input.agent, deadlineMs: input.deadlineMs }, });