diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a7290d79..efcff60c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,5 +71,15 @@ jobs: cargo test --locked -p temperpaw --quiet cargo test --locked -p paw-codex-worker --quiet cargo test --manifest-path os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml --quiet + # os-app WASM modules are their own workspaces, so `-p temperpaw` does + # not reach them. These five carry the OTS trajectory contract — the + # emitter that reads the record, the two writers that produce it, and + # the gate that fails when the temper pin gains the JCS fields. A gate + # nothing runs is not a gate (ADR-0035 section 17). + cargo test --manifest-path os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml --quiet + cargo test --manifest-path os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml --quiet + cargo test --manifest-path os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml --quiet + cargo test --manifest-path os-apps/paw-agent/wasm/openai-chat-wire/Cargo.toml --quiet + cargo test --manifest-path os-apps/paw-agent/wasm/monty_repl/Cargo.toml --quiet - name: Dashboard build run: cd dashboard && npm run build diff --git a/Cargo.lock b/Cargo.lock index 3b2cff193..84f49b016 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5939,7 +5939,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "temper-actor-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "anyhow", "async-trait", @@ -5962,7 +5962,7 @@ dependencies = [ [[package]] name = "temper-authz" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "cedar-policy", "opentelemetry", @@ -5976,7 +5976,7 @@ dependencies = [ [[package]] name = "temper-evolution" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "chrono", "serde", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "temper-jit" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "temper-observe" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "chrono", "opentelemetry", @@ -6028,7 +6028,7 @@ dependencies = [ [[package]] name = "temper-odata" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "axum 0.8.9", "chrono", @@ -6042,7 +6042,7 @@ dependencies = [ [[package]] name = "temper-optimize" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", @@ -6054,10 +6054,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "temper-ots" +version = "0.1.0" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" +dependencies = [ + "chrono", + "serde", + "serde_json", + "temper-runtime", + "uuid", +] + [[package]] name = "temper-platform" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "anyhow", "async-trait", @@ -6089,7 +6101,7 @@ dependencies = [ [[package]] name = "temper-runtime" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "anyhow", "chrono", @@ -6105,7 +6117,7 @@ dependencies = [ [[package]] name = "temper-sandbox" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "anyhow", "monty", @@ -6119,7 +6131,7 @@ dependencies = [ [[package]] name = "temper-server" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "aes-gcm", "async-stream", @@ -6149,6 +6161,7 @@ dependencies = [ "temper-jit", "temper-observe", "temper-odata", + "temper-ots", "temper-runtime", "temper-sandbox", "temper-spec", @@ -6172,7 +6185,7 @@ dependencies = [ [[package]] name = "temper-spec" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "quick-xml", "serde", @@ -6185,7 +6198,7 @@ dependencies = [ [[package]] name = "temper-store-postgres" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "chrono", "opentelemetry", @@ -6203,7 +6216,7 @@ dependencies = [ [[package]] name = "temper-store-redis" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "chrono", "fred", @@ -6220,7 +6233,7 @@ dependencies = [ [[package]] name = "temper-store-turso" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "libsql", "opentelemetry", @@ -6236,7 +6249,7 @@ dependencies = [ [[package]] name = "temper-verify" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "proptest", "serde", @@ -6250,7 +6263,7 @@ dependencies = [ [[package]] name = "temper-wasm" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "async-stream", "async-trait", diff --git a/crates/paw-codex-worker/Cargo.toml b/crates/paw-codex-worker/Cargo.toml index 600bd48e1..9df519605 100644 --- a/crates/paw-codex-worker/Cargo.toml +++ b/crates/paw-codex-worker/Cargo.toml @@ -18,7 +18,7 @@ libc = "0.2" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { workspace = true } serde_json = { workspace = true } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/temperpaw/Cargo.toml b/crates/temperpaw/Cargo.toml index 8c5446567..3977a2403 100644 --- a/crates/temperpaw/Cargo.toml +++ b/crates/temperpaw/Cargo.toml @@ -12,14 +12,14 @@ path = "src/main.rs" [dependencies] # Temper platform engine -temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d", features = ["observe"] } -temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } -temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-platform = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-observe = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-server = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b", features = ["observe"] } +temper-runtime = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-jit = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-authz = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-store-postgres = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } +temper-store-turso = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } # Paw transport (local) paw-transport = { path = "../paw-transport" } diff --git a/crates/temperpaw/tests/datadog_observability_contract.rs b/crates/temperpaw/tests/datadog_observability_contract.rs index e4c6294b6..257e1524a 100644 --- a/crates/temperpaw/tests/datadog_observability_contract.rs +++ b/crates/temperpaw/tests/datadog_observability_contract.rs @@ -59,13 +59,17 @@ fn collect_cargo_manifests(root: &Path, relative_dir: &Path, files: &mut Vec PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn session_spec() -> String { + fs::read_to_string(repo_root().join("os-apps/paw-agent/specs/session.ioa.toml")) + .expect("session.ioa.toml should exist") +} + +/// Byte offset just past the `}` that closes the first `{` in `source`. +/// +/// Braces inside string and char literals do not count — `format!("{id}")` is +/// balanced but a `"{"` would not be. Returns `None` when the braces never +/// balance, so a caller fails instead of silently widening its window. +fn closing_brace(source: &str) -> Option { + let bytes = source.as_bytes(); + let mut depth = 0usize; + let mut index = 0usize; + while index < bytes.len() { + match bytes[index] { + b'"' => { + index += 1; + while index < bytes.len() && bytes[index] != b'"' { + index += if bytes[index] == b'\\' { 2 } else { 1 }; + } + } + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Some(index + 1); + } + } + _ => {} + } + index += 1; + } + None +} + +fn emitter_source() -> String { + fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/emit_ots_trajectory/src/ots_build.rs"), + ) + .expect("ots_build.rs should exist") +} + +/// The production defect this track fixes: with span persistence off, every +/// stored trajectory carried an empty `decisions` array. +#[test] +fn run_tools_persists_tool_spans() { + let spec = session_spec(); + assert!( + spec.contains("persist_tool_spans_file = \"true\""), + "run_tools must persist tool spans; without them OTS decisions lose \ + tool wall-clock time and externalized turns lose their only evidence" + ); + assert!( + !spec.contains("persist_tool_spans_file = \"false\""), + "no trigger may disable tool-span persistence" + ); +} + +/// `metadata.spec_version` has to name the spec that actually ran, and the WASM +/// guest context exposes no spec hash — so the literal in the spec is the +/// identity, and it must track the app manifest. +#[test] +fn emitter_spec_version_matches_the_app_manifest() { + let manifest = fs::read_to_string(repo_root().join("os-apps/paw-agent/app.toml")) + .expect("paw-agent app.toml should exist"); + let version = manifest + .lines() + .find_map(|line| line.trim().strip_prefix("version = ")) + .map(|value| value.trim().trim_matches('"').to_string()) + .expect("app.toml should declare a version"); + let expected = format!("spec_version = \"paw-agent@{version}\""); + + assert!( + session_spec().contains(&expected), + "emit_ots_trajectory config must declare {expected}; the emitter reports \ + it as OTSMetadata.spec_version and it may not drift from app.toml" + ); +} + +/// Every terminal path that finishes a session has to emit a trajectory — +/// otherwise the training set silently loses whole classes of run. +#[test] +fn every_terminal_action_emits_a_trajectory() { + let spec = session_spec(); + let emit = "{ type = \"trigger\", name = \"emit_ots_trajectory\" }"; + for action in [ + "FinalizeResult", + "FinalizeResultNoReply", + "RecordResult", + "RecordResultNoReply", + "RecordResultInlineReply", + "Fail", + "Cancel", + "TimeoutFail", + ] { + let marker = format!("name = \"{action}\"\n"); + let start = spec + .find(&marker) + .unwrap_or_else(|| panic!("{action} must exist in session.ioa.toml")); + let block = &spec[start..]; + let block = &block[..block.find("\n[[action]]").unwrap_or(block.len())]; + assert!( + block.contains(emit), + "{action} must trigger emit_ots_trajectory" + ); + } +} + +/// Field names are the wire contract with the kernel's `temper-ots` structs. +/// A rename on either side silently drops the data at deserialization. +#[test] +fn emitter_uses_the_kernel_ots_field_names() { + let source = emitter_source(); + for field in [ + // OTSTrajectory / OTSMetadata + "\"trajectory_id\"", + "\"version\"", + "\"metadata\"", + "\"turns\"", + "\"task_description\"", + "\"timestamp_start\"", + "\"timestamp_end\"", + "\"agent_id\"", + "\"outcome\"", + // ARN-109 additive metadata + "\"harness\"", + "\"spec_version\"", + // OTSTurn + "\"turn_id\"", + "\"span_id\"", + "\"timestamp\"", + "\"messages\"", + "\"decisions\"", + // ARN-109 additive turn fields + "\"prompt_token_ids\"", + "\"completion_token_ids\"", + "\"response_mask\"", + "\"logprobs\"", + // OTSMessage + "\"message_id\"", + "\"role\"", + "\"content\"", + "\"reasoning\"", + // OTSDecision + "\"decision_id\"", + "\"decision_type\"", + "\"cause_id\"", + "\"choice\"", + "\"consequence\"", + "\"result_summary\"", + "\"error_type\"", + ] { + assert!( + source.contains(field), + "emitter must produce the OTS field {field}" + ); + } + + assert!( + source.contains("\"tool_selection\""), + "decision_type serializes snake_case per temper-ots enums" + ); + assert!( + source.contains("pub const HARNESS: &str = \"temperpaw\""), + "metadata.harness identifies the runtime that produced the run" + ); +} + +/// Inlining message bodies once cost ~300MB of a 491MB database (.proofs/061). +/// The budget constants are the guard, and the file-reference path is what +/// keeps large bodies out of the document entirely. +#[test] +fn emitter_bounds_inline_payloads() { + let source = emitter_source(); + assert!( + source.contains("MAX_MESSAGE_INLINE_CHARS"), + "per-message inline ceiling must exist" + ); + assert!( + source.contains("MAX_TRAJECTORY_INLINE_CHARS"), + "whole-document inline ceiling must exist" + ); + assert!( + source.contains("content_file_id"), + "externalized bodies must be referenced by file id, never inlined" + ); +} + +/// Retry idempotency: the Turso row is keyed on trajectory_id, so the id has to +/// be derived from the session and repeated inside metadata for the POST handler. +#[test] +fn emitter_keeps_trajectory_id_idempotency() { + let lib = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs"), + ) + .expect("emit_ots_trajectory lib.rs should exist"); + assert!( + lib.contains("format!(\"trj-{session_id}\")"), + "trajectory_id must stay derived from the session id" + ); + assert!( + lib.contains("MarkTrajectoryEmitted") && lib.contains("TrajectoryEmissionFailed"), + "emission status actions must stay wired" + ); + assert!( + emitter_source().contains("\"trajectory_id\": trajectory_id"), + "metadata must repeat trajectory_id for the server-side POST handler" + ); + + let spec = session_spec(); + for action in [ + "MarkTrajectoryEmitted", + "TrajectoryEmissionFailed", + "RetryTrajectoryEmission", + ] { + assert!( + spec.contains(&format!("name = \"{action}\"")), + "{action} must stay declared on the Session automaton" + ); + } +} + +/// A trajectory is written once and marked emitted. Emitting one built from a +/// transcript the emitter could not read stores a permanently incomplete row +/// that retry will never repair, so an unreadable transcript has to fail the +/// emission instead of degrading it. +#[test] +fn emitter_fails_closed_when_the_transcript_cannot_be_read() { + let lib = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs"), + ) + .expect("emit_ots_trajectory lib.rs should exist"); + + let read_call = lib + .find("match read_session_transcript(") + .expect("the emitter must read the session transcript"); + let error_arm = lib[read_call..] + .find("Err(error) => {") + .map(|offset| read_call + offset) + .expect("the transcript read must handle its error case"); + // The arm ends where its brace closes. Counting braces rather than matching + // a literal keeps this from breaking on an indentation change — but string + // literals hold braces too (`format!("{session_id}")`), so they are blanked + // first. An unbalanced scan fails rather than falling back to the whole + // file, where the strings asserted below all appear somewhere. + let arm = &lib[error_arm..]; + let end = closing_brace(arm).expect("the transcript error arm must close"); + let arm = &arm[..end]; + assert!( + arm.len() < 2_000, + "the extracted arm is {} bytes — the scan lost its bounds, and the \ + assertions below would then be reading the rest of the file", + arm.len() + ); + + assert!( + arm.contains("TrajectoryEmissionFailed"), + "an unreadable transcript must record a failed emission, not a partial trajectory" + ); + assert!( + arm.contains("return Ok(())"), + "an unreadable transcript must stop before the POST" + ); + assert!( + !arm.contains("String::new()"), + "substituting an empty transcript stores a spans-only row that is marked \ + emitted and therefore never repaired" + ); +} + +/// The read error is only half the problem. The shared TemperFS reader maps a +/// missing file to `Ok("")`, so a transcript that is *gone* arrives looking +/// exactly like a first-turn session that has not written one — and the +/// spans-only document built from it would be stored as complete. +#[test] +fn emitter_marks_an_absent_transcript_degraded_rather_than_complete() { + let helpers = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs")) + .expect("wasm-helpers lib.rs should exist"); + assert!( + helpers.contains("pub fn read_session_transcript(") + && helpers.contains("pub enum TranscriptPresence"), + "the transcript reader must report whether the transcript was there" + ); + assert!( + helpers.contains("fn read_temperfs_value_or_absent("), + "a 404 must be distinguishable from a 200 with an empty body" + ); + + let lib = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs"), + ) + .expect("emit_ots_trajectory lib.rs should exist"); + assert!( + lib.contains("\"emitted_degraded\""), + "a trajectory built without its evidence must not report plain 'emitted'" + ); + assert!( + lib.contains("ots_build::degradations(&trajectory)"), + "the entity status must be derived from the document that was stored, \ + so the row and the Session cannot disagree" + ); + + // A propagated error leaves trajectory_emission_status at "pending", which + // the sweep for failed emissions does not look at. The trigger declares no + // on_failure, and a kernel callback could not set the field anyway — its + // params are error / error_message / integration / duration_ms. + assert!( + lib.contains("if let Err(error) = emit()") + && lib.contains("\"trajectory_emission_status\": \"failed\""), + "every failure the guest can observe must record a failed emission, not \ + propagate and leave the status pending" + ); + + let emitter = emitter_source(); + assert!( + emitter.contains("pub const DEGRADED_TAG_PREFIX"), + "what a trajectory is missing must be named in the document" + ); + assert!( + emitter.contains("fn build_trajectory_marks_an_absent_transcript_as_degraded"), + "every absence reason must be proven to reach the stored document" + ); + assert!( + emitter.contains("fn build_trajectory_marks_an_unparseable_transcript_as_degraded"), + "arrival is not completeness: a transcript that is there but does not parse \ + is missing history too, and skipped lines are what make that invisible" + ); + + let spec = session_spec(); + let start = spec + .find("name = \"MarkTrajectoryEmitted\"") + .expect("MarkTrajectoryEmitted must exist"); + let block = &spec[start..]; + let block = &block[..block.find("\n[[action]]").unwrap_or(block.len())]; + assert!( + block.contains("trajectory_emission_error"), + "a degraded emission must record what was missing on the entity too" + ); +} + +/// The JCS contract fields ride natively on the pinned kernel structs, so a +/// consumer that deserializes a stored row and writes it back keeps them. The +/// carriers that stood in for them before the pin bump are gone and must stay +/// gone — a mirror that outlives its reason is a second source of truth. +/// Degradation markers are the exception: the kernel models no field for those, +/// so `metadata.tags` still carries them. +#[test] +fn emitter_carries_unmodeled_signal_in_kernel_modeled_fields() { + let emitter = emitter_source(); + assert!( + emitter.contains("fn kernel_round_trip_keeps_the_jcs_contract_fields"), + "the native round trip is what the carriers were removed against; it \ + has to be proven, not assumed" + ); + assert!( + emitter.contains("fn degradation_markers_survive_the_kernel_round_trip"), + "a completeness marker that a re-serialization drops is worse than none" + ); + for gone in [ + "TOKEN_SIGNAL_CARRIER_TYPE", + "TOKEN_SIGNALS_TAG", + "HARNESS_TAG_PREFIX", + "SPEC_VERSION_TAG_PREFIX", + ] { + assert!( + !emitter.contains(gone), + "{gone} is a pre-bump carrier. The kernel models these fields now, \ + so mirroring them again would leave the row carrying the same fact \ + twice with nothing keeping the copies equal" + ); + } + + // `-p temperpaw` does not reach the os-app WASM modules — they are their + // own workspaces — so a gate living in one of them only fires if CI runs + // that manifest. Asserting the source text of a test nothing executes + // proves nothing. + let ci = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("ci.yml should exist"); + for module in [ + "emit_ots_trajectory", + "provider_response_applier", + "wasm-helpers", + "openai-chat-wire", + "monty_repl", + ] { + assert!( + ci.contains(&format!( + "cargo test --manifest-path os-apps/paw-agent/wasm/{module}/Cargo.toml" + )), + "CI must run {module}'s tests; the OTS contract is asserted there" + ); + } + + let manifest = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml"), + ) + .expect("emit_ots_trajectory Cargo.toml should exist"); + // Dependency lines only: the comments around them name these crates too, + // and matching one of those would read as "no rev pinned". + let pinned_rev = |crate_name: &str| -> String { + manifest + .lines() + .map(str::trim) + .filter(|line| !line.starts_with('#')) + .find(|line| line.starts_with(crate_name) && line.contains("rev = \"")) + .and_then(|line| line.split("rev = \"").nth(1)) + .and_then(|rest| rest.split('"').next()) + .unwrap_or_else(|| panic!("{crate_name} should pin a rev")) + .to_string() + }; + let sdk_rev = pinned_rev("temper-wasm-sdk"); + let ots_rev = pinned_rev("temper-ots"); + assert_eq!( + sdk_rev, ots_rev, + "the round trip only proves anything if it runs against the kernel this \ + module is built for" + ); +} + +/// A span append that fails leaves `tool_spans_file_id` empty, which reads +/// exactly like a session that called no tools. The calls happened and their +/// record did not, so the difference has to reach the entity and the document. +#[test] +fn a_failed_span_write_is_recorded_rather_than_looking_like_no_tool_calls() { + let spec = session_spec(); + assert!( + spec.contains("name = \"tool_spans_write_failed\""), + "the Session must model the failure; nothing else distinguishes it" + ); + let carriers = spec + .matches("\"tool_spans_file_id\", \"tool_spans_write_failed\"") + .count(); + assert!( + carriers >= 4, + "every action that carries tool_spans_file_id must carry the failure \ + flag with it, or the flag never reaches the entity (found {carriers})" + ); + + let repl = fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/monty_repl/src/lib.rs")) + .expect("monty_repl lib.rs should exist"); + assert!( + repl.contains("params[\"tool_spans_write_failed\"] = json!(\"true\")"), + "the writer must record the failure, not only log it" + ); + + let emitter = emitter_source(); + assert!( + emitter.contains("tool_spans_write_failed"), + "the emitter must read the flag back and degrade on it" + ); + assert!( + emitter.contains("fn build_trajectory_marks_a_failed_span_write_as_degraded"), + "the degradation must be proven to reach the stored document" + ); +} + +/// A tool-span document that is already at its ceiling without a seal keeps +/// swallowing spans on every later batch. Nothing downstream can see that: the +/// document parses clean, so the trajectory built from it claims to hold every +/// tool call the session made. +#[test] +fn a_full_span_document_seals_itself_before_it_starts_dropping_spans() { + let session = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/monty_repl/src/session.rs")) + .expect("monty_repl session.rs should exist"); + assert!( + session.contains("fn encode_tool_spans_jsonl_seals_a_full_document_it_refuses_to_grow"), + "the seal on the refuse-to-grow path must be tested" + ); + assert!( + session.contains( + "fn encode_tool_spans_jsonl_does_not_seal_a_full_document_with_nothing_to_add" + ), + "sealing a document that dropped nothing would mark a complete run partial" + ); +} + +/// The truncation marker is a two-sided contract kept as two independent +/// literals: `monty_repl` seals a full span document with it, and the emitter +/// recognizes it instead of turning it into a decision. Nothing tied them +/// together, so a rename on one side would leave the other reading a real tool +/// call named `_tool_spans_truncated` — a decision the agent never made, +/// attributed to it, on a record that no longer knows it is partial. +#[test] +fn the_span_truncation_marker_means_the_same_thing_on_both_sides() { + const DECL: &str = "pub const TOOL_SPANS_TRUNCATED_MARKER: &str = \""; + + fn marker_literal(source: &str, whose: &str) -> String { + let start = source + .find(DECL) + .unwrap_or_else(|| panic!("{whose} must declare TOOL_SPANS_TRUNCATED_MARKER")); + let rest = &source[start + DECL.len()..]; + let end = rest + .find('"') + .unwrap_or_else(|| panic!("{whose}'s marker literal must terminate")); + rest[..end].to_string() + } + + let writer = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/monty_repl/src/session.rs")) + .expect("monty_repl session.rs should exist"); + let written = marker_literal(&writer, "the writer"); + let read = marker_literal(&emitter_source(), "the emitter"); + + assert_eq!( + written, read, + "the writer seals with {written:?} and the emitter looks for {read:?}; a \ + document sealed by one would be invisible to the other" + ); + + // The line the writer appends has to carry that same value as its + // `tool_name`, because that field — not a substring search — is what the + // emitter matches on. + assert!( + writer.contains(&format!("\\\"tool_name\\\":\\\"{written}\\\"")), + "the sealing line must carry the marker as its tool_name" + ); +} + +/// Token-level signals scale with completion length. Bounding each one on its +/// own does not bound their sum, and the entry's `extra_json` ceiling is +/// enforced by the kernel on the whole value: cross it and the per-turn facts +/// the emitter needs are replaced along with the signals. +#[test] +fn token_signals_are_bounded_against_their_aggregate_ceilings() { + let applier = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs"), + ) + .expect("provider_response_applier lib.rs should exist"); + assert!( + applier.contains("MAX_ENTRY_EXTRA_BYTES"), + "the SessionEntry writer must bound the whole extra_json value, not only each signal" + ); + assert!( + applier.contains("fn assistant_turn_extra_bounds_signals_against_the_entry_ceiling") + && applier.contains("fn entry_extra_ceiling_matches_the_session_entry_spec"), + "the aggregate ceiling must be tested, and pinned to the spec that declares it" + ); + assert!( + applier.contains("fn entry_extra_budget_counts_escaped_bytes"), + "extra_json is a string-typed field, so the budget must count the bytes the \ + kernel measures — the escaped encoding, not the raw one" + ); + + // Choosing which signal to sacrifice is policy; the ceiling itself is an + // invariant, and writers with no policy of their own (the JSONL sync path + // re-materializing pre-bound extras) reach the same field. + let helpers = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs")) + .expect("wasm-helpers lib.rs should exist"); + assert!( + helpers.contains("pub const MAX_ENTRY_EXTRA_BYTES") + && helpers.contains("fn bound_entry_extra") + && helpers.contains("fn entry_extra_is_bounded_at_the_write_boundary"), + "the entry ceiling must be enforced at the boundary every writer passes through" + ); + + let wire = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs")) + .expect("openai-chat-wire lib.rs should exist"); + assert!( + wire.contains("fn merge_token_signals_rejects_non_numeric_token_arrays"), + "token ids and mask bits come from a per-agent configurable endpoint; \ + non-numeric elements must be rejected at capture, not sized as if numeric" + ); + + // Completion-side signals accumulate across events, so a signal taken twice + // from one event is stored twice — and with a single signal present there + // is no second array to disagree on length, so nothing downstream detects + // it. Every accumulator must collapse an event's levels before merging. + // This holds the rule for the three that exist and for the next one added. + let caller = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/provider_caller/src/lib.rs")) + .expect("provider_caller lib.rs should exist"); + for (source, whose) in [(&wire, "openai-chat-wire"), (&caller, "provider_caller")] { + for (index, line) in source.lines().enumerate() { + let call = line.trim_start(); + if call.starts_with("merge_token_signals(&mut self.") { + assert!( + call.ends_with("&source);"), + "{whose}:{} merges a raw event level straight into an \ + accumulator. Route it through event_token_signals first, or \ + one event's signals get stored twice: {call}", + index + 1 + ); + } + } + } + for test in [ + "fn one_event_contributes_each_token_signal_once", + "fn openrouter_event_contributes_each_token_signal_once", + "fn openai_response_completed_contributes_each_token_signal_once", + ] { + assert!( + wire.contains(test) || caller.contains(test), + "each accumulator needs its own repeated-payload test ({test})" + ); + } + + let emitter = emitter_source(); + assert!( + emitter.contains("pub const MAX_TOKEN_SIGNAL_BYTES"), + "the trajectory must bound the one payload its character budgets do not" + ); + assert!( + emitter.contains("fn build_trajectory_bounds_token_signals_across_the_document"), + "the trajectory-wide signal ceiling must be tested" + ); +} + +/// Tool-call ids are unique only within a turn: providers that omit them get +/// synthetic ones that restart at every response. Anything keyed on them across +/// a session collapses two calls into one. +#[test] +fn tool_call_ids_survive_provider_fallbacks() { + let wire = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs")) + .expect("openai-chat-wire lib.rs should exist"); + assert!( + wire.contains("pub fn synthetic_tool_call_id("), + "the fallback id must be built in one place so every provider scopes it" + ); + for source in [ + "os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs", + "os-apps/paw-agent/wasm/provider_caller/src/lib.rs", + ] { + let text = fs::read_to_string(repo_root().join(source)) + .unwrap_or_else(|_| panic!("{source} should exist")); + assert!( + !text.contains("format!(\"tool_{}\", idx + 1)") + && !text.contains("format!(\"or_tool_{}\", idx + 1)"), + "{source} must not mint a turn-local tool call id" + ); + } + + let emitter = emitter_source(); + assert!( + emitter.contains("fn claim_span("), + "the emitter must match spans to calls per turn, not through a \ + document-wide id index" + ); + assert!( + !emitter.contains("span_by_id"), + "a document-wide id -> span map lets a later turn overwrite an earlier one" + ); +} + +/// Serde ignores unknown fields, so deserializing the emitted document into the +/// kernel structs proves nothing about the fields the kernel does not model. +/// Those are enumerated and asserted separately. +#[test] +fn emitter_pins_the_fields_the_kernel_does_not_model() { + let emitter = emitter_source(); + assert!( + emitter.contains("KERNEL_UNMODELED_FIELDS"), + "the extensions the pinned kernel drops on a round trip must be named" + ); + assert!( + emitter.contains("fn kernel_round_trip_drops_exactly_the_unmodeled_extensions"), + "the unmodeled set must be asserted, so modeling one of them is noticed" + ); + assert!( + emitter.contains("fn rows_without_the_new_fields_still_deserialize"), + "an old-row fixture must prove the additions stayed additive" + ); +} + +/// Per-turn timestamps and token counts come from the entry itself, because the +/// entity event log is a hot tail that drops older events at snapshot boundaries. +#[test] +fn session_entries_carry_their_own_wall_clock() { + let helpers = + fs::read_to_string(repo_root().join("os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs")) + .expect("wasm-helpers lib.rs should exist"); + assert!( + helpers.contains("fn stamp_recorded_at"), + "every SessionEntry must be stamped with its own creation time" + ); + assert!( + helpers.contains("\"ts_ms\""), + "the stamp field is ts_ms; the emitter reads it back" + ); + + let applier = fs::read_to_string( + repo_root().join("os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs"), + ) + .expect("provider_response_applier lib.rs should exist"); + assert!( + applier.contains("fn assistant_turn_extra"), + "assistant entries must record per-turn provider, model and usage facts" + ); + for field in ["input_tokens", "output_tokens", "stop_reason"] { + assert!( + applier.contains(&format!("\"{field}\"")), + "assistant turn extras must record {field}" + ); + } +} diff --git a/crates/temperpaw/tests/paw_fs_hot_path.rs b/crates/temperpaw/tests/paw_fs_hot_path.rs index 50fe4e400..150cf9969 100644 --- a/crates/temperpaw/tests/paw_fs_hot_path.rs +++ b/crates/temperpaw/tests/paw_fs_hot_path.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const EXPECTED_TEMPER_REV: &str = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d"; +const EXPECTED_TEMPER_REV: &str = "a747f7d40cb556371168f8460bc72806c3574d2b"; const OLD_TEMPER_REV: &str = "c584a52b59924e66502576646f50131b0d763a2a"; fn repo_root() -> PathBuf { diff --git a/crates/temperpaw/tests/session_turn_architecture.rs b/crates/temperpaw/tests/session_turn_architecture.rs index 7047ebcbb..edb32977a 100644 --- a/crates/temperpaw/tests/session_turn_architecture.rs +++ b/crates/temperpaw/tests/session_turn_architecture.rs @@ -331,9 +331,10 @@ fn first_turn_session_entries_materialize_after_provider_success() { ); assert!( helpers.contains("fn session_entries_materialized") - && helpers.contains("return Ok(String::new());") + && helpers.contains("presence: TranscriptPresence::PendingFirstTurn,") && helpers.contains("virtual first-turn SessionEntries ref"), - "virtual first-turn SessionEntries reads should return empty JSONL without listing SessionEntries" + "virtual first-turn SessionEntries reads should return empty JSONL without listing \ + SessionEntries, reported as pending rather than as a transcript that was read" ); assert!( applier.contains("materialize_initial_session_entries_with_assistant"), @@ -701,7 +702,7 @@ fn record_result_clears_pending_tool_state_on_terminal_completion() { assert!( spec.contains( - "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" + "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"tool_spans_write_failed\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" ), "RecordResult should be able to clear pending tool and approval fields on completion" ); @@ -760,7 +761,7 @@ fn record_result_no_reply_preserves_terminal_cleanup_without_delivery_trigger() let action_block = &action_tail[..action_end]; assert!( action_block.contains( - "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" + "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"tool_spans_write_failed\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" ), "RecordResultNoReply should keep RecordResult cleanup/accounting params" ); @@ -830,7 +831,7 @@ fn record_result_inline_reply_preserves_channel_audit_without_agent_reply() { let action_block = &action_tail[..action_end]; assert!( action_block.contains( - "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" + "params = [\"result\", \"conversation\", \"input_tokens\", \"output_tokens\", \"session_leaf_id\", \"session_entries_materialized\", \"repl_file_id\", \"tool_spans_file_id\", \"tool_spans_write_failed\", \"system_prompt_hash\", \"system_prompt_file_id\", \"provider_response_file_id\", \"provider_response_inline_json\", \"pending_tool_calls\", \"pending_tool_context\", \"pending_decision_id\", \"reply_attachments_json\"]" ), "RecordResultInlineReply should keep RecordResult cleanup/accounting params" ); diff --git a/docs/adrs/0035-ots-trajectory-emission.md b/docs/adrs/0035-ots-trajectory-emission.md index 3979cef3e..1e1130c88 100644 --- a/docs/adrs/0035-ots-trajectory-emission.md +++ b/docs/adrs/0035-ots-trajectory-emission.md @@ -2,6 +2,7 @@ **Status:** Accepted **Date:** 2026-04-16 +**Amended:** 2026-08-11 — sections 9-18 (ARN-109: real turns, decisions, content, and the completeness of what is stored) **Related:** ADR-0005 (Temper-Native Orchestration), ADR-0015 (Convergence Analyst), ADR-0022 (LLM Calling Infrastructure Optimizations), ADR-0032 (TemperFS Agent Operations), ADR-0034 (Bounded Session Context and LLM Turn Decomposition) ## Context @@ -87,13 +88,29 @@ Serialization is snake_case per `temper-ots/src/models/enums.rs:22-29` (the enum Emission failures surface as a state change on the Session entity via three new fields: - `trajectory_id` (string) — generated once before first POST, reused on retry for idempotency (`INSERT OR REPLACE` on the Turso side is keyed on this) -- `trajectory_emission_status` (string, initial `"pending"`) — transitions to `"emitted"` or `"failed"` -- `trajectory_emission_error` (string) — last error message +- `trajectory_emission_status` (string, initial `"pending"`) — transitions to `"emitted"`, `"emitted_degraded"` (stored, but built without some of its evidence — see section 16) or `"failed"` +- `trajectory_emission_error` (string) — last error message, or the evidence a degraded emission was missing Three new self-loop actions from `Completed | Failed | Cancelled`: -- `MarkTrajectoryEmitted(trajectory_id)` — success path -- `TrajectoryEmissionFailed(error)` — failure path, also fired via the integration's `on_failure` hook -- `RetryTrajectoryEmission` — guarded by `trajectory_emission_status == "failed" AND retry_count < 1` +- `MarkTrajectoryEmitted(trajectory_id, status, error)` — success path, degraded or not +- `TrajectoryEmissionFailed(error, status)` — failure path +- `RetryTrajectoryEmission` — guarded by `trajectory_retry_count < 1`, so it is a + one-shot manual retry regardless of which status the last attempt recorded + +The guest dispatches both itself. It does not lean on an `on_failure` hook, and +the trigger declares none: a WASM callback receives `error`, `error_message` and +`integration`, and no effect kind sets a string field to a literal — so a hook +could not write `trajectory_emission_status` at all and would leave it at +`"pending"`, which the sweep for failed emissions does not look at, while +`error_message` (a Session state variable) would be overwritten with a +trajectory error in place of the session's own recorded failure reason. Every failure the guest can +observe (transport error on either read, non-2xx from the POST) therefore routes +through `TrajectoryEmissionFailed` with `trajectory_emission_status = "failed"`. +What remains outside its reach is a guest trap or timeout, where the module +never runs; with no `on_failure` declared the platform surfaces that as +`temper_integration_failure_dropped_total` plus an `integration_failure_dropped` +Observe event (ADR-0152), and the row stays `"pending"`. A sweep should treat a +terminal session still at `"pending"` as unemitted for that reason. Retry is one-shot and state-machine-visible, not in-WASM retry loops. Beyond one retry, the Evolution Engine can sweep `trajectory_emission_status = "failed"` rows as an I-Record in a future track. @@ -101,6 +118,327 @@ Retry is one-shot and state-machine-visible, not in-WASM retry loops. Beyond one The Convergence Analyst session already has access to the `temper_get_trajectories` MCP tool. `handle_probe_done` assembles a `temper.get_trajectories(...)` fetch-instruction block per probe_agent_id and injects it into the analyst's `user_message`. Fetch-on-demand keeps the probe→analyst data path within the governed Temper API surface rather than inline splicing large JSON into the 32KB-ceiling callback param. +### 9. Turns come from the SessionEntry tree (amends section 1, 2026-08-11) + +Section 1 shipped one synthetic turn per session and deferred real turn +boundaries. In production that produced trajectories with a single turn, no +messages, and an empty `decisions` array — a row that no evaluation agent and no +RL consumer can use. The deferral is now closed. + +The emitter reads the session transcript (the `session_file_id` reference, which +resolves either to a TemperFS JSONL file or to the SessionEntry rows) and walks +the chain from the recorded `session_leaf_id` to the root. Each assistant entry +closes one LLM cycle, so it opens a turn; the user, tool-result, steering, and +compaction entries that precede it are that turn's prompt side. The final turn +may have no assistant entry — that is a session interrupted mid-cycle, and it is +kept rather than discarded. + +When the leaf is missing or its parent chain is broken (continuation and +recovery races can push the Session field ahead of durable rows), the emitter +falls back to the newest walkable entry, then to raw file order. A damaged tree +degrades the trajectory; it does not empty it. + +`turn_count` is not the turn source — it is recorded as `_session_turn_count` so +a consumer can see when the reconstructed count disagrees with the counter the +state machine kept. + +### 10. Decisions are reconstructed from the transcript, with spans as enrichment (amends section 4) + +Section 2 made the tool-span JSONL the sole input to decisions, and section 4 +mapped one span to one decision. That made a single config flag +(`persist_tool_spans_file`, shipped as `"false"`) sufficient to empty every +stored trajectory, which is what happened. Decisions now have two independent +sources: + +- **The transcript.** `tool_use` blocks on the assistant entry give the tool + name and arguments; the `tool_result` blocks that land on the next turn give + success and result text. Both are already persisted for the model's own + benefit, so this path costs nothing extra and cannot be switched off. +- **The spans.** They supply wall-clock duration, and they are the only evidence + left when a message body was externalized to TemperFS. Spans nothing claims + still become decisions rather than being dropped. + +`cause_id` is set to the `tool_call_id` on every decision. It is the join +between a decision and the observation it caused — the `tool_result` block +carrying the same id, which by construction sits on the following turn. + +Span persistence is enabled in the spec and defaults to ON in the guest: a +missing config key must not silently cost the training data. The cost this +guards against is real (the span document is rewritten in full on every tool +batch), so span records are compacted before persistence — results capped at 600 +characters, arguments at 2000 — and the document is capped at 256KB with an +explicit truncation marker. + +### 11. Message content is referenced, not inlined + +Inlining message bodies once cost roughly 300MB of a 491MB database +(`.proofs/061`). Trajectories are stored as opaque blobs and are written once +per session, so the same failure is available here. + +Bodies that already live in TemperFS are emitted as file references +(`content_file_id`, `content_file_version_id`) and never fetched. Inline text is +bounded twice: 4000 characters per message and 64000 characters per trajectory, +with the dropped character count recorded so a consumer can tell truncation from +absence. Tool arguments over 4000 serialized characters collapse to a preview +plus the original size. The session's own artifacts (session tree, tool spans, +prepared context, provider response, system prompt) are listed as OTS context +resources, which gives consumers the pointers without the payloads. + +### 12. Spec identity and harness + +`metadata.harness` is `"temperpaw"` — the runtime that produced the run, which a +cross-harness training set has to distinguish. + +`metadata.spec_version` identifies the actor spec the run executed under. The +WASM guest context exposes only config, trigger params, entity state, and ids +(`temper-wasm-sdk::Context`); it carries no spec hash, and asking the server for +one would add an HTTP round trip to every terminal transition. So the identity +is declared in the spec's own trigger config as `@` and travels +with the spec that declares it. A repo contract test pins that literal to +`os-apps/paw-agent/app.toml`, so the two cannot drift apart silently. + +Alternatives rejected: an extra request to read the installed-app version (a +round trip per session for a value the spec already knows), and a hand-written +hash literal (drifts the moment someone forgets to update it). + +### 13. Token counts always, token ids only when the serving stack sends them + +Per-turn prompt and completion token counts come from the provider response and +are recorded on the assistant entry when it is written, so they are exact rather +than reconstructed. They surface as `_prompt_tokens` and `_completion_tokens`; +session totals surface as `_token_usage`. The OTS schema has no field for token +counts, and the underscore prefix marks non-standard fields the same way +`_duration_ms` already does on decisions. + +`prompt_token_ids`, `completion_token_ids`, `response_mask`, and `logprobs` are +emitted with exactly those names when the pipeline recorded them, and are absent +otherwise. RL consumers need token ids because retokenizing text drifts, but no +provider is asked for them: the OpenAI-compatible and Responses stream parsers +capture them if the server streams them (flattening OpenAI's +`logprobs.content[].logprob` shape to the flat array the contract requires), the +Anthropic Messages stream carries none, and nothing issues a second request. +Malformed signals are dropped rather than passed through — a fabricated mask is +worse than a missing one. + +Per-turn timestamps have the same shape of problem. The entity event log is a +hot tail that drops older events at snapshot boundaries, so it cannot date a +long session's turns. Every SessionEntry is therefore stamped with its own +`ts_ms` at creation, and the event log is only a fallback for entries written +before that stamp existed. + +### 14. Tool-call ids are only unique within a turn (2026-08-11) + +Providers that omit tool-call ids get a synthetic one, and the synthetic id used +to be positional (`tool_1`, `or_tool_1`), restarting with every response. Two +turns that each made one call therefore shared an id, and the emitter's +document-wide `id -> span` and `id -> observation` maps let the second call +overwrite the first: both decisions reported the second call's result, error +flag and duration. + +Both halves are fixed. The synthetic id is now scoped by the provider's own +response id (`chatcmpl-…_tool_1`), and the conversion of transcript history back +to chat format scopes its fallback by message position, so one request cannot +carry the same call id twice. Independently of that, the emitter attributes +observations and spans **per turn**: observations parsed from turn K's prompt +answer turn K-1, and spans are claimed by position — the first `tool_1` span +goes to the first `tool_1` call. Repeated ids therefore cost nothing even in +rows written before the id change, and a model that reuses an id cannot collapse +two decisions into one. + +### 15. Signals that are positionally aligned travel as a set (2026-08-11) + +`completion_token_ids`, `response_mask` and `logprobs` are indexed by generated +token: element *i* of each describes the same token. A payload assembled from +partial data breaks that silently, and a consumer has no way to detect it. + +Two gates. The OpenAI-compatible parser flattens a `logprobs.content[]` payload +only when **every** entry carries a numeric `logprob`, rejecting the payload +whole rather than skipping the bad entry and shortening the array. The emitter +then refuses to write the completion-side signals unless the ones present agree +on length, recording `_token_signals_misaligned` with the observed lengths so +the drop is visible. Prompt-side ids do not index into the completion and are +unaffected. + +The truncation marker has the same shape of problem. Whether a span document was +sealed is decided from the reserved `tool_name` of a parsed record, never a +substring search — a tool that reads or greps this source returns the marker +literal in its own result, and that must not make a complete run look partial. +The seal check also stopped slicing the document at a byte offset, which trapped +the guest on any multibyte tail. + +### 16. An unreadable transcript fails the emission; an absent one degrades it (2026-08-11) + +A trajectory is written once and the session is then marked emitted. Emitting a +spans-only document because the transcript read returned 503 or a policy denial +would store a permanently incomplete row that no retry ever repairs, because the +session no longer looks failed. + +A transcript read **error** therefore records `TrajectoryEmissionFailed` and +stops before the POST, leaving the row absent and `RetryTrajectoryEmission` (and +the Evolution Engine sweep) able to produce a complete one. + +An **absent** transcript is a different thing and still emits, because a retry +cannot restore a transcript that is not there. It must not pass as complete +either. The shared reader mapped every "nothing" case to `Ok("")` — a legacy +404, a 200 with an empty body, a SessionEntries query with no rows, and a +first-turn session that has not materialized any — so the emitter could not tell +a session that never wrote history from one whose history is gone, and stored +both as if the turn structure had simply not existed. + +`read_session_transcript` now returns the transcript together with a +`TranscriptPresence`, and each non-present reason (`missing_file`, `empty_file`, +`no_entries`, `pending_first_turn`, `undeclared`) reaches the stored document. +The same applies to a declared tool-span file that 404s. + +Arrival is not the test, either. Every skip-and-continue in the reconstruction +path is a way for a short record to look whole, so each one now reports: + +- `transcript_unparseable` — `parse_session_entries` skips lines that do not + parse, deliberately, so one corrupted line cannot cost a whole trajectory. The + count of skipped lines travels with the document. +- `transcript_leaf_unresolved` — the recorded `session_leaf_id` is the session's + own claim about where its history ends. When it does not resolve, the fallback + chain of section 9 is an older leaf, so the *newest* turns are exactly what is + missing. That is the shape a half-written final turn takes. A cyclic ancestry + counts as unresolved rather than as a chain that stopped early: everything + above the loop is unreachable, so the fragment is not the leaf's history. +- `transcript_no_turns` — entries parsed but produced no turn, which yields the + same synthetic single-turn document an empty transcript does. +- `tool_spans_unparseable` — `parse_tool_span_document` skips malformed span + lines for the same reason, and each one is a tool call whose only evidence is + gone. +- `token_signals_dropped` — a signal the SessionEntry writer or the trajectory + budget refused. Both record the size they dropped; the tag is what makes the + loss visible on the Session rather than only inside the document. + +Without these, corruption, a stale leaf, or a partially written span append each +reach the same false-complete row that a 404 used to. + +A degraded document carries the reason three ways, because each survives a +different consumer: + +- `metadata.tags` gets `degraded:` — kernel-modeled, so it survives a + consumer that re-serializes the row through `OTSTrajectory`. A completeness + marker is the last thing that may be lost on a round trip: losing it turns a + partial record into an apparently whole one. +- The document carries `_transcript` / `_tool_spans_missing` for a consumer + reading the raw row. +- The Session reports `trajectory_emission_status = "emitted_degraded"` and + names the missing evidence in `trajectory_emission_error`, so a sweep can find + degraded rows without opening them. The status is derived from the document + that was actually stored, so the entity and the row cannot disagree. + +`turn_count` is deliberately not one of the checks. It counts continuations — +tool results, steering, plan resumes — not assistant messages, so it does not +equal the reconstructed turn count even on a healthy session; comparing them +would mark nearly every trajectory degraded and make the marker worthless. It +travels as `_session_turn_count` (section 9) for a consumer that wants to weigh +the two. + +The single-retry guard is unchanged: it counts retries, not statuses, so a +degraded emission neither consumes nor triggers one. + +### 17. The JCS contract fields ride natively (2026-08-12, supersedes the interim carriers) + +`metadata.harness`, `metadata.spec_version`, the per-turn token-level RL signals +and `decisions[].cause_id` are the JCS contract fields. The pin now sits on +temper `a747f7d4` — the merge of nerdsane/temper#416 — where `temper-ots` +declares every one of them as an optional additive field. They ride natively: a +consumer that deserializes a stored row into `OTSTrajectory` and writes it back +keeps them, and `kernel_round_trip_keeps_the_jcs_contract_fields` asserts each +one survives with its value intact, through typed struct access rather than JSON +shape alone. + +`metadata.trajectory_id` stays unmodeled, by design and not by omission: the +server's POST handler reads it from there before any struct is involved, and the +kernel models the top-level `trajectory_id` that mirrors it. +`kernel_round_trip_drops_exactly_the_unmodeled_extensions` now pins that as the +only dropped field, so a new extension cannot appear unnoticed. + +**What this replaced.** Until the bump, the pinned structs modelled none of +these, and serde ignores unknown fields, so each one travelled through a field +the kernel did model: `cause_id` mirroring the modelled `decision_id`; run +provenance repeated in `metadata.tags` as `harness:` / `spec_version:`; and the +token-level signals summarised as an inventory in `context.entities[]` +(`turn_token_signals`) plus a `token_signals:present` tag. The signal arrays +themselves always stayed on the turn, under the names the kernel now declares — +copying megabyte-scale arrays into a carrier would have reproduced the payload +failure section 11 exists to prevent — so the bump was a deletion rather than a +migration, which is what it turned out to be. + +Those carriers are gone. A test failing is what removed them: the gate asserted +each contract field was still dropped, so the bump made it fail and its message +named the removal list. The contract test now asserts the carrier constants are +absent, because a mirror that outlives its reason is a second source of truth +with nothing keeping the copies equal. + +Degradation markers are the exception and stay in `metadata.tags`: the kernel +models no field for "what this record was built without", and losing that marker +turns a partial row into an apparently whole one (section 16). + +### 18. Token-level signals are bounded twice (2026-08-11) + +These arrays scale with completion length and are the only payload the +character budgets of section 11 do not touch, so they are bounded where they are +written and again where they are read. + +At **capture**, arrays streamed under `prompt_token_ids`, +`completion_token_ids` and `response_mask` are accepted only when every element +is a number. The OpenAI-compatible endpoint is configurable per agent, so what +arrives under those names is not trusted; text there would be unbounded foreign +content sized against a budget that assumes numbers, and the emitter's own shape +checks would drop it from the trajectory regardless. + +Capture also counts each signal once per event. Completion-side signals +accumulate across events, so a server carrying the same payload at both levels +of a single event — `usage` and `choices[0]` in a chat chunk, `response` and +`response.usage` in a Responses `response.completed` — would have it stored +twice, and when only one signal is present nothing downstream can detect that: +there is no second array to disagree on length. All three stream accumulators +collapse an event's levels through one shared `event_token_signals` before +merging, the content level winning over the accounting one. A contract test +refuses any accumulator that merges a raw event level directly, so a fourth wire +shape cannot reintroduce it. + +On the **SessionEntry**, `extra_json` declares +`overflow_inline_max_bytes = 131072`; past it the kernel replaces or +externalizes the *whole* field, which would take the per-turn facts — `ts_ms`, +provider, model, token counts — along with the signals that caused the overflow. +Bounding each signal at 32KiB does not bound their sum: four signals just under +that ceiling each pass and cross the entry ceiling together. The writer +therefore spends a running budget and names what did not fit as +`_dropped_bytes`. The budget counts bytes the way the kernel does — +`extra_json` is a string-typed state variable, so the ceiling applies to the +JSON encoded again as a JSON string, and counting the unescaped length would +under-measure a quote-dense value. A test pins the constant to the spec that +declares it, and another pins the measurement to that double encoding. + +Choosing *which* signal to sacrifice is policy and belongs to the writer that +knows what the signals mean. The ceiling itself is an invariant, so it is also +enforced at the single boundary every writer passes through +(`session_entry_create_body` in `wasm-helpers`), which drops the largest +non-essential members until the value fits and leaves `_dropped_bytes` +behind. That covers writers with no signal policy of their own — in particular +the JSONL sync path, which re-materializes extras written before any of these +bounds existed. The per-turn facts are the last thing it sacrifices, and when +they are themselves what does not fit (an oversized `stop_reason`, or so many +members that the drop markers alone hold the value over) it shortens them and +keeps a single count rather than returning a value over the ceiling: returning +one costs the entire field, which is the outcome the bound exists to prevent. +Members are measured once and dropped largest-first, because re-measuring per +drop is quadratic on an object a corrupted line can make wide. + +The refusals travel forward. `_dropped_bytes` written at capture is read +back by the emitter into the same `_token_signals_dropped` record a +trajectory-budget drop produces, so a turn whose signals were refused before the +emitter saw them is distinguishable from a provider that sent none. + +In the **trajectory**, signals are bounded at 1MiB across the whole document, +spent in turn order, with drops recorded as `_token_signals_dropped` on the turn +and `degraded:token_signals_dropped` on the row. A dropped signal that leaves a +trace is debuggable; a silent one reads as a turn the serving stack never +produced signals for. + ## Consequences ### Positive @@ -133,6 +471,47 @@ Per the repository's mandatory red-green TDD and end-to-end proof requirements: The foresight meta-loop behavioural rerun (Run 011) is explicitly deferred — that proof happens on main after merge, in a separate foresight run tracked separately from this ADR. +### Verification of the 2026-08-11 amendment (ARN-109) + +- **Round trip against the kernel structs** — `emit_ots_trajectory` takes + `temper-ots` as a host-only dev-dependency and deserializes its own output + into `OTSTrajectory`, asserting the reconstructed turns, message roles, + content types, decision types, and durations. A field-name or type drift on + either side fails the build instead of storing an unreadable row. Terminal + states other than success round-trip too. Because serde ignores unknown + fields, the extensions the kernel does not model are pinned separately by + `kernel_round_trip_drops_exactly_the_unmodeled_extensions` and + `pinned_kernel_still_lacks_the_jcs_contract_fields`; their kernel-modeled + carriers are proven lossless by + `token_signal_inventory_survives_the_kernel_round_trip` and + `degradation_markers_survive_the_kernel_round_trip`; and an old-row fixture + proves the additions stayed additive (decision section 17). +- **Unit tests** — turn reconstruction from a two-cycle transcript, leaf + fallback and parent-cycle guards, decision/observation pairing with + `cause_id`, per-message and per-trajectory inline budgets, oversized tool + arguments, externalized-body references, token-signal validation, timestamp + derivation, and the RFC-3339 conversion. +- **Repo contract tests** — `crates/temperpaw/tests/ots_trajectory_contract.rs` + pins span persistence to on, `spec_version` to `app.toml`, the OTS field names + the kernel deserializes, the inline budget, trajectory-id idempotency, the + requirement that every terminal action still emits, that an unreadable + transcript fails the emission rather than degrading it, that an *absent* one + is marked degraded rather than stored as complete, that the unmodeled fields + travel through kernel-modeled carriers, that the round-trip runs against the + same kernel revision the guest is built for, that both token-signal ceilings + exist, and that no provider mints a turn-local tool-call id. +- **Bounded-write tests** — `monty_repl` span compaction and the span-file size + ceiling, including that a truncated document still parses line by line, that + multibyte tool output does not trap the seal check, and that a tool result + quoting the marker does not seal the document. +- **Guest build** — every touched module rebuilt for `wasm32-unknown-unknown` + (`monty_repl` for `wasm32-wasip1`); the `temper-ots` dev-dependency is never + part of a guest build. + +The live local end-to-end run (`scripts/prove_track3_ots.py` against a local +temper-server with the paw-agent app installed and real provider credentials) +gates the deploy and is recorded on the pull request, not here. + ## Rejected Alternatives ### 1. Server-side converter in Temper @@ -158,3 +537,23 @@ See Decision section 5. Requires LLM prompt change; separate track. ### 6. Extend Temper's OTS schema with openpaw-specific fields Rejected. The OTS schema is a shared platform contract (`temper-ots` crate). openpaw-specific metadata, if any, can live in `metadata.tags` without schema changes. + +### 7. Fetch externalized message bodies at emission time (2026-08-11) + +Rejected. A session can externalize many entries, so this is N TemperFS reads on +a terminal transition, and it puts the full bodies back into the stored blob — +the exact failure `.proofs/061` records. The emitter references the files +instead; a consumer that wants a body can read it through the governed API. + +### 8. Read the installed-app version for `spec_version` (2026-08-11) + +Rejected. An HTTP round trip per terminal session to learn a value the spec +already knows. See decision section 12. + +### 9. Plumb a token-id request flag into provider calls (2026-08-11) + +Rejected for this track. Asking providers for logprobs or token ids changes the +request, costs latency and money on every turn, and most providers in this stack +cannot return them at all. The emitter carries the fields when the serving stack +volunteers them and leaves them absent otherwise; turning them on deliberately +for a training run is a separate decision with its own cost analysis. diff --git a/os-apps/paw-agent/specs/model.csdl.xml b/os-apps/paw-agent/specs/model.csdl.xml index 49d84bc04..1b851ecd4 100644 --- a/os-apps/paw-agent/specs/model.csdl.xml +++ b/os-apps/paw-agent/specs/model.csdl.xml @@ -539,6 +539,7 @@ + @@ -560,6 +561,7 @@ + @@ -581,6 +583,7 @@ + diff --git a/os-apps/paw-agent/specs/session.ioa.toml b/os-apps/paw-agent/specs/session.ioa.toml index a8a747fc1..35a6950e0 100644 --- a/os-apps/paw-agent/specs/session.ioa.toml +++ b/os-apps/paw-agent/specs/session.ioa.toml @@ -439,6 +439,16 @@ name = "tool_spans_file_id" type = "string" initial = "" +# Set to "true" the first time a tool-span append fails. An empty +# tool_spans_file_id reads identically whether the session made no tool calls or +# made them and lost the record, and the emitter has to tell those apart. Never +# cleared: a later successful append does not undo the earlier loss. + +[[state]] +name = "tool_spans_write_failed" +type = "string" +initial = "" + # --- State Variables: OTS Trajectory Emission (ADR-0035) --- [[state]] @@ -866,7 +876,11 @@ max_sync_file_bytes = "61440" max_sync_files = "64" sync_exclude = "__pycache__,node_modules,.git,.next,dist,build,target,coverage,venv,.venv" normal_repl_state_max_bytes = "524288" -persist_tool_spans_file = "false" +# Tool spans are the only record of tool-call wall-clock time and the input the +# OTS emitter needs to complete each decision. Disabling this leaves every +# stored trajectory with an empty decisions array (ARN-109). The span document +# is compacted and size-capped in monty_repl, so the cost stays bounded. +persist_tool_spans_file = "true" dd_api_key = "{secret:dd_api_key}" dd_app_key = "{secret:dd_app_key}" dd_site = "{secret:dd_site}" @@ -882,7 +896,7 @@ name = "HandleToolResults" kind = "input" from = ["Executing"] to = "PreparingContext" -params = ["pending_tool_calls", "conversation", "session_leaf_id", "repl_file_id", "tool_spans_file_id", "sandbox_url", "sandbox_id", "sandbox_provider", "system_prompt_hash", "system_prompt_file_id", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] +params = ["pending_tool_calls", "conversation", "session_leaf_id", "repl_file_id", "tool_spans_file_id", "tool_spans_write_failed", "sandbox_url", "sandbox_id", "sandbox_provider", "system_prompt_hash", "system_prompt_file_id", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] hint = "Tool results received. Increment turn, transition to PreparingContext, and assemble the next bounded provider turn." effect = [ { type = "increment", var = "turn_count" }, @@ -1094,6 +1108,11 @@ module = "emit_ots_trajectory" [action.triggers.config] temper_api_url = "{secret:temper_api_url}" +# Identity of the spec this run executed under, emitted as +# OTSMetadata.spec_version. The WASM guest context exposes no spec hash, so the +# identity travels with the spec that declares it; `ots_trajectory_contract` +# pins this literal to os-apps/paw-agent/app.toml so it cannot drift. +spec_version = "paw-agent@0.1.0" [[action]] @@ -1215,7 +1234,7 @@ name = "RecordResult" kind = "input" from = ["ApplyingProviderResponse", "Executing"] to = "Completed" -params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] +params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "tool_spans_write_failed", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] hint = "Session complete. Triggered by LLM end_turn or agent calling temper.done(result)." effect = [ { type = "increment", var = "input_tokens" }, @@ -1230,7 +1249,7 @@ name = "RecordResultNoReply" kind = "input" from = ["ApplyingProviderResponse"] to = "Completed" -params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] +params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "tool_spans_write_failed", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] hint = "Direct no-route Session complete. Set result and emit trajectory without invoking terminal reply delivery." effect = [ { type = "increment", var = "input_tokens" }, @@ -1244,7 +1263,7 @@ name = "RecordResultInlineReply" kind = "input" from = ["ApplyingProviderResponse"] to = "Completed" -params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] +params = ["result", "conversation", "input_tokens", "output_tokens", "session_leaf_id", "session_entries_materialized", "repl_file_id", "tool_spans_file_id", "tool_spans_write_failed", "system_prompt_hash", "system_prompt_file_id", "provider_response_file_id", "provider_response_inline_json", "pending_tool_calls", "pending_tool_context", "pending_decision_id", "reply_attachments_json"] hint = "Inline CLI/TUI reply already recorded on the Channel. Set result and emit trajectory without invoking terminal reply delivery." effect = [ { type = "increment", var = "input_tokens" }, @@ -1318,16 +1337,23 @@ hint = "Reply delivery failed after the session reached a terminal state." # --- Actions: OTS Trajectory Emission (ADR-0035) --- # The emit_ots_trajectory integration dispatches MarkTrajectoryEmitted on success -# (passing both trajectory_id and trajectory_emission_status="emitted" as params, -# which the framework applies as field updates). On failure, the integration's -# on_failure hook fires TrajectoryEmissionFailed which records the error. +# (passing trajectory_id, trajectory_emission_status and trajectory_emission_error +# as params, which the framework applies as field updates). A trajectory built +# without its transcript or its tool spans is still stored — a retry cannot +# restore evidence that is gone — so it reports status "emitted_degraded" and +# names what is missing in trajectory_emission_error. On failure the guest +# dispatches TrajectoryEmissionFailed itself, rather than relying on an +# on_failure hook: a WASM callback receives error / error_message / integration, +# and no effect kind sets a string field to a literal, so a hook could not write +# trajectory_emission_status and would leave it at "pending" — while writing +# error_message over the session's own recorded failure reason. [[action]] name = "MarkTrajectoryEmitted" kind = "input" from = ["Completed", "Failed", "Cancelled"] -params = ["trajectory_id", "trajectory_emission_status"] -hint = "emit_ots_trajectory succeeded — record the trajectory_id and mark emission complete." +params = ["trajectory_id", "trajectory_emission_status", "trajectory_emission_error"] +hint = "emit_ots_trajectory succeeded — record the trajectory_id and mark emission complete ('emitted', or 'emitted_degraded' with the missing evidence named in trajectory_emission_error)." [[action]] name = "TrajectoryEmissionFailed" diff --git a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml index 655d5c64d..4d0451b48 100644 --- a/os-apps/paw-agent/wasm/agent_reply/Cargo.toml +++ b/os-apps/paw-agent/wasm/agent_reply/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml index d65ea206b..7a6db2dd2 100644 --- a/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml +++ b/os-apps/paw-agent/wasm/coding_agent_runner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock index c7f48fdc8..279e310b7 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.lock @@ -121,7 +121,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml index 225f16a30..7cd20d84c 100644 --- a/os-apps/paw-agent/wasm/context_compactor/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_compactor/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock index 1f131e0d1..49d9ae834 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.lock +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.lock @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml index c451a6e06..706b92bd4 100644 --- a/os-apps/paw-agent/wasm/context_preparer/Cargo.toml +++ b/os-apps/paw-agent/wasm/context_preparer/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml index c1f40f600..765fc00c8 100644 --- a/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml +++ b/os-apps/paw-agent/wasm/cron_compute_next/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock index 94fc37ffd..8887dc39b 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.lock @@ -2,27 +2,287 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "emit-ots-trajectory" version = "0.1.0" dependencies = [ "serde_json", + "temper-ots", "temper-wasm-sdk", "wasm-helpers", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -41,6 +301,33 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.228" @@ -68,7 +355,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -84,6 +371,53 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "syn" version = "2.0.117" @@ -95,21 +429,243 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "temper-ots" +version = "0.1.0" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" +dependencies = [ + "chrono", + "serde", + "serde_json", + "temper-runtime", + "uuid", +] + +[[package]] +name = "temper-runtime" +version = "0.1.0" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" +dependencies = [ + "anyhow", + "chrono", + "serde", + "serde_json", + "thiserror", + "tokio", + "toml", + "tracing", + "uuid", +] + [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", ] +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + [[package]] name = "wasm-helpers" version = "0.1.0" @@ -118,6 +674,83 @@ dependencies = [ "temper-wasm-sdk", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml index ab616383f..aed3f01b7 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/Cargo.toml @@ -9,6 +9,26 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" + +# Host-only. The emitted JSON is round-tripped through the kernel's real OTS +# structs, so a drift in a field the kernel models fails here instead of +# silently dropping data at the /api/ots/trajectories boundary. This rev must +# stay equal to the temper-wasm-sdk rev above — a round trip against a kernel +# the guest is not built for proves nothing, and a contract test asserts it. +# +# What the round trip cannot prove: serde ignores unknown fields, so an +# extension the kernel does not model passes it untouched. Those fields are +# listed in `ots_build::KERNEL_UNMODELED_FIELDS` and asserted separately by +# `kernel_round_trip_drops_exactly_the_unmodeled_extensions`, which fails the +# day the kernel starts modeling one of them. Never built for wasm32. +# +# This rev (a747f7d4, the merge of nerdsane/temper#416) is the one that added +# the JCS contract fields to `temper-ots`, so they ride natively and the +# carriers that stood in for them are gone. Rolling the pin back below it would +# fail `kernel_round_trip_keeps_the_jcs_contract_fields` — see ADR-0035 +# section 17. +[dev-dependencies] +temper-ots = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs b/os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs index 594fddded..add3c1752 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/src/lib.rs @@ -7,10 +7,15 @@ use serde_json::json; use temper_wasm_sdk::prelude::*; -use wasm_helpers::{entity_field_str, resolve_temper_api_url, runtime_headers}; +use wasm_helpers::{ + TranscriptPresence, entity_field_str, error_excerpt, read_session_transcript, + resolve_temper_api_url, runtime_headers, +}; mod ots_build; +use ots_build::TrajectoryInputs; + #[unsafe(no_mangle)] pub extern "C" fn run(_ctx_ptr: i32, _ctx_len: i32) -> i32 { let result = (|| -> Result<(), String> { @@ -28,70 +33,185 @@ pub extern "C" fn run(_ctx_ptr: i32, _ctx_len: i32) -> i32 { .to_string(); if !matches!(status.as_str(), "Completed" | "Failed" | "Cancelled") { + // Not an emission failure: nothing was owed yet, and no status + // field should claim otherwise. return Err(format!( "emit_ots_trajectory: session {session_id} not in terminal state (status={status})" )); } - let agent_id = fields - .get("agent_id") - .and_then(|v| v.as_str()) - .unwrap_or(session_id.as_str()) - .to_string(); + // Everything past this point is emission, and every way it can fail has + // to leave `trajectory_emission_status = "failed"` on the Session. + // Propagating an error instead would leave the field at "pending", and a + // "pending" row is invisible to the sweep for failed emissions, so the + // trajectory would never be retried. An `on_failure` hook cannot stand + // in for this: the kernel passes a WASM callback `error`, + // `error_message` and `integration`, and no effect kind sets a string + // field to a literal, so the hook could not write + // `trajectory_emission_status` at all — while `error_message` *is* a + // Session state variable, so it would overwrite the session's own + // recorded failure reason with this one. + let emit = || -> Result<(), String> { + let agent_id = fields + .get("agent_id") + .and_then(|v| v.as_str()) + .unwrap_or(session_id.as_str()) + .to_string(); - // Stable trajectory_id across retries — enables INSERT OR REPLACE idempotency. - let existing_trajectory_id = fields - .get("trajectory_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let trajectory_id = if existing_trajectory_id.is_empty() { - format!("trj-{session_id}") - } else { - existing_trajectory_id.to_string() - }; + // Stable trajectory_id across retries — enables INSERT OR REPLACE idempotency. + let existing_trajectory_id = fields + .get("trajectory_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let trajectory_id = if existing_trajectory_id.is_empty() { + format!("trj-{session_id}") + } else { + existing_trajectory_id.to_string() + }; - let tool_spans_file_id = fields - .get("tool_spans_file_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let tool_spans_jsonl = - read_temperfs_file_safe(&ctx, &temper_api_url, &tenant, tool_spans_file_id)?; - - let trajectory = ots_build::build_trajectory( - &trajectory_id, - &session_id, - &agent_id, - &status, - &fields, - &tool_spans_jsonl, - &ctx.entity_state, - ); + let tool_spans_file_id = fields + .get("tool_spans_file_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + // A declared span file that 404s is missing evidence, not an absence of + // tool calls, and the trajectory has to say so. + let tool_spans_read = + read_temperfs_file_safe(&ctx, &temper_api_url, &tenant, tool_spans_file_id)?; + let tool_spans_missing = !tool_spans_file_id.is_empty() && tool_spans_read.is_none(); + let tool_spans_jsonl = tool_spans_read.unwrap_or_default(); - let body = trajectory.to_string(); - let url = format!("{temper_api_url}/api/ots/trajectories"); - let mut headers = runtime_headers( - &ctx, - &tenant, - &fields, - Some("application/json"), - Some("application/json"), - ); - headers.push(("X-Agent-Id".to_string(), agent_id.clone())); - headers.push(("X-Session-Id".to_string(), session_id.clone())); - headers.push(("X-Tenant-Id".to_string(), tenant.clone())); - headers.push(("X-Trajectory-Id".to_string(), trajectory_id.clone())); - - let resp = ctx.http_call("POST", &url, &headers, &body)?; - if !(200..300).contains(&resp.status) { - let msg = format!( - "POST /api/ots/trajectories failed (HTTP {}): {}", - resp.status, - truncate_body(&resp.body) + // The transcript is the source of real turn boundaries. A read failure + // is not a reason to store a spans-only row: the trajectory would be + // permanently incomplete and, being marked emitted, never repaired. It + // is recorded as a failed emission instead, which leaves the row absent + // and the retry path (`RetryTrajectoryEmission`, plus the Evolution + // Engine sweep) able to produce a complete one. An absent transcript is + // a different thing from an unreadable one and still emits — a + // first-turn session has no materialized entries yet — but the document + // is spans-only, and it says so rather than passing as complete. + let session_file_id = fields + .get("session_file_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let (session_jsonl, transcript) = if session_file_id.is_empty() { + (String::new(), TranscriptPresence::Undeclared) + } else { + match read_session_transcript( + &ctx, + &temper_api_url, + &tenant, + &fields, + session_file_id, + ) { + Ok(read) => (read.jsonl, read.presence), + Err(error) => { + let msg = format!( + "session transcript read failed for {session_id}; no trajectory emitted so a retry can produce a complete one: {error}" + ); + ctx.log("warn", &format!("emit_ots_trajectory: {msg}")); + set_success_result( + "TrajectoryEmissionFailed", + &json!({ + "trajectory_emission_error": msg, + "trajectory_emission_status": "failed", + }), + ); + return Ok(()); + } + } + }; + + let spec_version = resolve_spec_version(&ctx); + + let trajectory = ots_build::build_trajectory(&TrajectoryInputs { + trajectory_id: &trajectory_id, + session_id: &session_id, + agent_id: &agent_id, + status: &status, + fields: &fields, + session_jsonl: &session_jsonl, + tool_spans_jsonl: &tool_spans_jsonl, + entity_state: &ctx.entity_state, + spec_version: &spec_version, + transcript, + tool_spans_missing, + }); + + // Degradations are decided from the same inputs the document was built + // from, so the entity and the row cannot disagree about them. + let degradations = ots_build::degradations(&trajectory); + if !degradations.is_empty() { + ctx.log( + "warn", + &format!( + "emit_ots_trajectory: session {session_id} produced a degraded trajectory ({})", + degradations.join(", ") + ), + ); + } + + let body = trajectory.to_string(); + let url = format!("{temper_api_url}/api/ots/trajectories"); + let mut headers = runtime_headers( + &ctx, + &tenant, + &fields, + Some("application/json"), + Some("application/json"), ); + headers.push(("X-Agent-Id".to_string(), agent_id.clone())); + headers.push(("X-Session-Id".to_string(), session_id.clone())); + headers.push(("X-Tenant-Id".to_string(), tenant.clone())); + headers.push(("X-Trajectory-Id".to_string(), trajectory_id.clone())); + + let resp = ctx.http_call("POST", &url, &headers, &body)?; + if !(200..300).contains(&resp.status) { + let msg = format!( + "POST /api/ots/trajectories failed (HTTP {}): {}", + resp.status, + truncate_body(&resp.body) + ); + ctx.log("warn", &format!("emit_ots_trajectory: {msg}")); + set_success_result( + "TrajectoryEmissionFailed", + &json!({ + "trajectory_emission_error": msg, + "trajectory_emission_status": "failed", + }), + ); + return Ok(()); + } + ctx.log( - "warn", - &format!("emit_ots_trajectory: {msg}"), + "info", + &format!( + "emit_ots_trajectory: emitted trajectory {trajectory_id} for session {session_id} (status={status})" + ), + ); + + // A degraded row is still a row — retrying cannot restore a transcript + // that is not there — so it is marked emitted, with what it is missing + // recorded on the entity as well as inside the document. + let (emission_status, emission_error) = if degradations.is_empty() { + ("emitted", String::new()) + } else { + ("emitted_degraded", degradations.join(",")) + }; + + set_success_result( + "MarkTrajectoryEmitted", + &json!({ + "trajectory_id": trajectory_id, + "trajectory_emission_status": emission_status, + "trajectory_emission_error": emission_error, + }), ); + Ok(()) + }; + + if let Err(error) = emit() { + let msg = format!("emit_ots_trajectory failed for {session_id}: {error}"); + ctx.log("warn", &msg); set_success_result( "TrajectoryEmissionFailed", &json!({ @@ -99,43 +219,46 @@ pub extern "C" fn run(_ctx_ptr: i32, _ctx_len: i32) -> i32 { "trajectory_emission_status": "failed", }), ); - return Ok(()); } - - ctx.log( - "info", - &format!( - "emit_ots_trajectory: emitted trajectory {trajectory_id} for session {session_id} (status={status})" - ), - ); - - set_success_result( - "MarkTrajectoryEmitted", - &json!({ - "trajectory_id": trajectory_id, - "trajectory_emission_status": "emitted", - }), - ); Ok(()) })(); if let Err(error) = result { - // Top-level errors still go through set_error_result so the platform's - // on_failure hook fires. These are panics / preflight failures, not - // remote HTTP errors (those route through set_success_result above). + // Only preflight failures reach here: no host context, or a session that + // is not terminal. Neither owes a trajectory, so neither may claim an + // emission status. A guest trap or timeout never reaches this code at + // all and surfaces as the platform's dropped-integration metric. set_error_result(&error); } 0 } +/// Identity of the actor spec this run executed under. +/// +/// The WASM guest context carries no spec hash (`temper-wasm-sdk::Context` +/// exposes config, trigger params, entity state and ids only — see ADR-0035 +/// decision section 9), so the governing identity is declared in the spec's own +/// trigger config as `@` and travels with the spec that declares +/// it. A repo contract test keeps that literal pinned to `app.toml`. +fn resolve_spec_version(ctx: &Context) -> String { + ctx.config + .get("spec_version") + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_default() +} + +/// Read a TemperFS file, reporting a missing one as `None` rather than as an +/// empty body — the caller has to be able to tell "no tool calls" from "the +/// record of the tool calls is gone". fn read_temperfs_file_safe( ctx: &Context, temper_api_url: &str, tenant: &str, file_id: &str, -) -> Result { +) -> Result, String> { if file_id.is_empty() { - return Ok(String::new()); + return Ok(Some(String::new())); } let fields = ctx .entity_state @@ -152,19 +275,46 @@ fn read_temperfs_file_safe( ); let resp = ctx.http_call("GET", &url, &headers, "")?; match resp.status { - 200 => Ok(resp.body), - 404 => Ok(String::new()), + 200 => Ok(Some(resp.body)), + 404 => Ok(None), other => Err(format!( "emit_ots_trajectory: TemperFS read failed (HTTP {other})" )), } } +/// A bounded excerpt of a failing response body. +/// +/// Cut by characters, never by byte offset: this runs on the path that reports +/// the failure, and a multibyte character straddling the cut would trap the +/// guest there — replacing the recorded failure with a dead module and a +/// Session still reading "pending". fn truncate_body(body: &str) -> String { const LIMIT: usize = 240; - if body.len() <= LIMIT { - body.to_string() + let excerpt = error_excerpt(body, LIMIT); + if excerpt.len() == body.len() { + excerpt } else { - format!("{}...", &body[..LIMIT]) + format!("{excerpt}...") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// This runs while reporting a failed POST. Cutting the body at a byte + /// offset traps the guest whenever a multibyte character straddles the cut, + /// so the failure report is replaced by a dead module and a Session still + /// reading "pending" — the exact outcome the failure path exists to avoid. + #[test] + fn truncate_body_survives_a_multibyte_body() { + let body = "é".repeat(400); + let truncated = truncate_body(&body); + assert!(truncated.ends_with("...")); + assert_eq!(truncated.trim_end_matches('.').chars().count(), 240); + + assert_eq!(truncate_body("short"), "short"); + assert_eq!(truncate_body(""), ""); } } diff --git a/os-apps/paw-agent/wasm/emit_ots_trajectory/src/ots_build.rs b/os-apps/paw-agent/wasm/emit_ots_trajectory/src/ots_build.rs index 77373a6dc..a8b6b8714 100644 --- a/os-apps/paw-agent/wasm/emit_ots_trajectory/src/ots_build.rs +++ b/os-apps/paw-agent/wasm/emit_ots_trajectory/src/ots_build.rs @@ -2,10 +2,111 @@ //! //! Kept separate from `lib.rs` so the mapping logic can be unit-tested without //! a live Temper context. The resulting JSON shape matches the `OTSTrajectory` -//! serde schema in `temper/crates/temper-ots/src/models/trajectory.rs` as of -//! 2026-04-16 and is versioned at "0.1.0". +//! serde schema in `temper/crates/temper-ots/src/models/trajectory.rs` and is +//! versioned at "0.1.0". +//! +//! Turn reconstruction (ARN-109): turns come from the SessionEntry tree, not +//! from a synthetic single turn. Every assistant entry closes one LLM cycle; +//! the user / tool-result / steering / compaction entries that precede it are +//! that turn's prompt side. Decisions come from the assistant's `tool_use` +//! blocks, are answered by the `tool_result` blocks of the following turn, and +//! are enriched with wall-clock duration from the per-session tool-span JSONL. +//! +//! Payload discipline: message bodies that already live in TemperFS are emitted +//! as file references, never inlined, and inline text is bounded per message and +//! per trajectory. Inlining full bodies once cost ~300MB of a 491MB database +//! (.proofs/061); the budget below is the guard against a repeat. + +use serde_json::{Map, Value, json}; +use std::collections::{BTreeMap, BTreeSet}; +use wasm_helpers::TranscriptPresence; + +/// Value of `metadata.harness` — identifies the runtime that produced the run. +pub const HARNESS: &str = "temperpaw"; +/// Tag prefix naming evidence this trajectory was built without. +/// +/// It lives in `metadata.tags` because that is kernel-modeled: a completeness +/// marker is the last thing that may vanish when a consumer re-serializes a +/// stored row, since losing it turns a partial record into an apparently +/// complete one. +pub const DEGRADED_TAG_PREFIX: &str = "degraded:"; +/// OTS schema version emitted by this module. +pub const OTS_VERSION: &str = "0.1.0"; + +// The JCS contract fields — `metadata.harness`, `metadata.spec_version`, the +// per-turn token-level RL signals and `decisions[].cause_id` — are modeled by +// the pinned `temper-ots` structs and ride natively. A consumer that +// deserializes a stored row into `OTSTrajectory` and writes it back keeps them. +// +// They did not always. Until the pin reached the revision that added them, each +// one travelled through a field the kernel did model — tag mirrors for the +// provenance pair, an inventory in `context.entities[]` for the signals — and a +// test failed the moment a bump made those carriers dead weight. That is what +// removed them; `kernel_round_trip_keeps_the_jcs_contract_fields` now holds the +// native path, and would fail if a carrier were reintroduced or the pin rolled +// back. +// +// `metadata.trajectory_id` stays unmodeled by design: the server's POST handler +// reads it from there before any struct is involved, and the kernel models the +// top-level `trajectory_id` that mirrors it. + +/// Largest inline text body attached to a single OTS message. +pub const MAX_MESSAGE_INLINE_CHARS: usize = 4_000; +/// Largest total inline text across the whole trajectory document. +pub const MAX_TRAJECTORY_INLINE_CHARS: usize = 64_000; +/// Largest `consequence.result_summary`. +pub const MAX_RESULT_SUMMARY_CHARS: usize = 500; +/// Largest serialized `choice.arguments` payload. +pub const MAX_ARGUMENTS_CHARS: usize = 4_000; +/// Largest inlined system prompt. +pub const MAX_SYSTEM_PROMPT_CHARS: usize = 2_000; +/// Largest task description taken from the user message. +pub const MAX_TASK_DESCRIPTION_CHARS: usize = 500; +/// Largest total serialized size of token-level signals in one trajectory. +/// +/// These arrays scale with completion length and are the only payload in the +/// document the character budgets above do not touch, so a long session could +/// otherwise carry many megabytes of them. Signals past the ceiling are dropped +/// visibly rather than silently. +pub const MAX_TOKEN_SIGNAL_BYTES: usize = 1_048_576; -use serde_json::{Value, json}; +/// Every token-level signal name, in the shape OTS turns use. The prompt-side +/// ids stand alone; the rest are the positionally aligned completion set. +/// `token_signal_fields_cover_every_signal` keeps the two in step. +pub const TOKEN_SIGNAL_FIELDS: &[&str] = &[ + "prompt_token_ids", + "completion_token_ids", + "response_mask", + "logprobs", +]; + +const EPOCH: &str = "1970-01-01T00:00:00Z"; + +/// Everything the emitter knows about a finished session. +pub struct TrajectoryInputs<'a> { + /// Stable trajectory id (`trj-`) — the idempotency key. + pub trajectory_id: &'a str, + /// Session entity id. + pub session_id: &'a str, + /// Owning agent entity id. + pub agent_id: &'a str, + /// Terminal session status (`Completed` / `Failed` / `Cancelled`). + pub status: &'a str, + /// Session entity `fields` object. + pub fields: &'a Value, + /// Session tree as JSONL (one entry per line), oldest first. + pub session_jsonl: &'a str, + /// Per-session tool-span JSONL written by `monty_repl`. + pub tool_spans_jsonl: &'a str, + /// Full entity state (used for the event log). + pub entity_state: &'a Value, + /// Identity of the governing actor spec (`@`). + pub spec_version: &'a str, + /// Whether the transcript behind `session_jsonl` was actually there. + pub transcript: TranscriptPresence, + /// Whether a declared tool-span file could not be found. + pub tool_spans_missing: bool, +} /// Map the Session's terminal state + has_result flag to an OTS `OutcomeType`. /// @@ -42,72 +143,542 @@ pub fn truncate_chars(s: &str, max_chars: usize) -> String { s.chars().take(max_chars).collect() } -/// Convert a single tool-span JSON object (as emitted by `monty_repl::emit_tool_call_telemetry`) -/// into an `OTSDecision` JSON value. +/// Format milliseconds since the Unix epoch as an RFC-3339 UTC timestamp. /// -/// Optional fields (state, alternatives, evaluation, credit_assignment, embedding) are -/// not populated — see ADR-0035 decision section 5. The `duration_ms` is preserved as -/// a non-standard `_duration_ms` field for evaluation-agent consumption. -pub fn span_to_decision(span: &Value) -> Value { - let tool_call_id = span - .get("tool_call_id") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - let tool_name = span - .get("tool_name") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - let is_error = span - .get("is_error") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - let result = span.get("result").and_then(|v| v.as_str()).unwrap_or(""); - let duration_ms = span.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(0); - - let mut choice = json!({ "action": tool_name }); - match span.get("arguments") { - Some(Value::String(s)) => { - if let Ok(parsed) = serde_json::from_str::(s) { - choice["arguments"] = parsed; - } else { - choice["arguments"] = Value::String(s.clone()); +/// The WASM guests have no chrono dependency, so this implements the civil-date +/// conversion directly. Negative inputs clamp to the epoch — a trajectory with a +/// pre-1970 timestamp is a corrupted clock, not signal worth preserving. +pub fn rfc3339_from_millis(millis: i64) -> String { + let millis = millis.max(0); + let total_secs = millis / 1_000; + let ms = (millis % 1_000) as u32; + let days = total_secs / 86_400; + let secs_of_day = total_secs % 86_400; + let (year, month, day) = civil_from_days(days); + let hour = secs_of_day / 3_600; + let minute = (secs_of_day % 3_600) / 60; + let second = secs_of_day % 60; + format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}.{ms:03}Z" + ) +} + +/// Howard Hinnant's `civil_from_days` — days since 1970-01-01 to (y, m, d). +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let y = if m <= 2 { y + 1 } else { y }; + (y, m, d) +} + +/// A parsed session-tree entry (JSONL line or materialized SessionEntry row). +#[derive(Debug, Clone)] +pub struct TreeEntry { + /// Entry id (`u-*`, `a-*`, `t-*`, `c-*`, `s-*`, `h-*`). + pub id: String, + /// Parent entry id, absent for the header. + pub parent_id: Option, + /// `header` | `message` | `compaction` | `steering`. + pub entry_type: String, + /// `user` | `assistant` | empty. + pub role: String, + /// Inline content when the entry was not externalized. + pub content: Option, + /// TemperFS file id when the body was externalized. + pub content_file_id: Option, + /// Immutable file version for stable historical reads. + pub content_file_version_id: Option, + /// Token estimate recorded with the entry. + pub tokens: u64, + /// The whole line, so extras (`ts_ms`, token signals, usage) stay reachable. + pub raw: Value, +} + +impl TreeEntry { + fn from_value(value: Value) -> Option { + let id = value.get("id").and_then(Value::as_str)?.to_string(); + if id.is_empty() { + return None; + } + Some(TreeEntry { + parent_id: value + .get("parentId") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string), + entry_type: value + .get("type") + .and_then(Value::as_str) + .unwrap_or("message") + .to_string(), + role: value + .get("role") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + content: value.get("content").cloned(), + content_file_id: value + .get("content_file_id") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string), + content_file_version_id: value + .get("content_file_version_id") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string), + tokens: value.get("tokens").and_then(Value::as_u64).unwrap_or(0), + id, + raw: value, + }) + } + + fn is_assistant(&self) -> bool { + self.role == "assistant" + } + + fn is_header(&self) -> bool { + self.entry_type == "header" + } + + /// Wall-clock time the entry was recorded, when the writer stamped one. + fn recorded_at(&self) -> Option { + if let Some(ms) = self.raw.get("ts_ms").and_then(json_i64) { + return Some(rfc3339_from_millis(ms)); + } + self.raw + .get("timestamp") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + } + + /// Content blocks when the entry carries an Anthropic-style block array. + fn blocks(&self) -> Option<&Vec> { + self.content.as_ref().and_then(Value::as_array) + } +} + +fn json_i64(value: &Value) -> Option { + value + .as_i64() + .or_else(|| value.as_f64().map(|f| f as i64)) + .or_else(|| value.as_str().and_then(|s| s.trim().parse::().ok())) +} + +fn json_u64(value: &Value) -> Option { + value + .as_u64() + .or_else(|| value.as_f64().filter(|f| *f >= 0.0).map(|f| f as u64)) + .or_else(|| value.as_str().and_then(|s| s.trim().parse::().ok())) +} + +/// Parse session-tree JSONL into ordered entries. Invalid lines are skipped — +/// a corrupted line must not cost the whole trajectory. +pub fn parse_session_entries(session_jsonl: &str) -> Vec { + session_jsonl + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter_map(TreeEntry::from_value) + .collect() +} + +/// A root→leaf chain, and whether the session's own leaf produced it. +pub struct ResolvedChain { + /// Entry indices from root to leaf. + pub chain: Vec, + /// False when a recorded `session_leaf_id` could not be walked and the + /// chain came from a fallback. The fallback keeps the trajectory usable, + /// but the newest turns are the ones missing from it, so a consumer has to + /// be told. An empty `session_leaf_id` claims nothing and stays true. + pub from_recorded_leaf: bool, +} + +/// Resolve the root→leaf chain the session actually executed. +/// +/// Prefers the recorded `session_leaf_id`. When that leaf is missing or its +/// parent chain is broken (continuation/recovery races can leave the Session +/// field ahead of durable rows), falls back to the newest walkable entry and +/// finally to raw file order, so a damaged tree still yields real turns. +pub fn resolve_chain(entries: &[TreeEntry], leaf_id: &str) -> ResolvedChain { + let mut by_id: BTreeMap<&str, usize> = BTreeMap::new(); + for (index, entry) in entries.iter().enumerate() { + by_id.insert(entry.id.as_str(), index); + } + + // `None` means this leaf did not yield a walkable chain — either an ancestor + // is missing, or the parent pointers loop. A cycle is not a chain that + // merely stops early: everything above the loop is unreachable, so treating + // the fragment as a resolved chain would report a truncated history as the + // whole one. + let walk = |leaf: &str| -> Option> { + let mut chain = Vec::new(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut cursor = Some(leaf.to_string()); + while let Some(id) = cursor { + let index = *by_id.get(id.as_str())?; + if !seen.insert(entries[index].id.as_str()) { + return None; // cycle guard — malformed parent pointer } + chain.push(index); + cursor = entries[index].parent_id.clone(); + } + chain.reverse(); + Some(chain) + }; + + let has_message = |chain: &[usize]| chain.iter().any(|i| !entries[*i].is_header()); + + if !leaf_id.is_empty() + && let Some(chain) = walk(leaf_id) + && has_message(&chain) + { + return ResolvedChain { + chain, + from_recorded_leaf: true, + }; + } + + // Try the newest entries only. Walking every entry would be quadratic on a + // long session, and a tree whose last hundred leaves are all unwalkable is + // damaged far past the point where a smarter search would help. + const FALLBACK_LEAF_ATTEMPTS: usize = 100; + let from_recorded_leaf = leaf_id.is_empty(); + for index in (0..entries.len()).rev().take(FALLBACK_LEAF_ATTEMPTS) { + if let Some(chain) = walk(&entries[index].id) + && has_message(&chain) + { + return ResolvedChain { + chain, + from_recorded_leaf, + }; + } + } + + // Raw file order is not a walk. No leaf — recorded or not — produced a + // chain with a message in it, so the parent structure is unusable and the + // order is a guess at what followed what. That holds whether or not a leaf + // was recorded, so it is reported even when none was: a session with no + // recorded leaf and dangling parents is still missing its shape. An empty + // transcript is the one case with nothing to have resolved, and the + // presence reasons already speak for it. + ResolvedChain { + chain: (0..entries.len()).collect(), + from_recorded_leaf: entries.is_empty(), + } +} + +/// One reconstructed LLM cycle: the prompt-side entries plus the assistant +/// entry that closed it. The final turn may have no assistant entry when the +/// session was cancelled or failed mid-cycle. +#[derive(Debug, Clone)] +pub struct TurnDraft { + /// Indices of prompt-side entries, in order. + pub prompt: Vec, + /// Index of the assistant entry that closed the turn. + pub assistant: Option, +} + +/// Group a root→leaf chain into turns at assistant-message boundaries. +pub fn group_turns(entries: &[TreeEntry], chain: &[usize]) -> Vec { + let mut turns: Vec = Vec::new(); + let mut prompt: Vec = Vec::new(); + + for index in chain { + let entry = &entries[*index]; + if entry.is_header() { + continue; + } + if entry.is_assistant() { + turns.push(TurnDraft { + prompt: std::mem::take(&mut prompt), + assistant: Some(*index), + }); + } else { + prompt.push(*index); } - Some(Value::Null) | None => {} - Some(other) => { - choice["arguments"] = other.clone(); + } + + if !prompt.is_empty() { + turns.push(TurnDraft { + prompt, + assistant: None, + }); + } + + turns +} + +/// Bounded inline-text accounting shared across the whole trajectory. +struct InlineBudget { + remaining: usize, +} + +impl InlineBudget { + fn new(total: usize) -> Self { + InlineBudget { remaining: total } + } + + /// Take up to `per_message_max` characters, respecting the global budget. + /// Returns the text plus the number of characters dropped. + fn take(&mut self, text: &str, per_message_max: usize) -> (String, usize) { + let cap = per_message_max.min(self.remaining); + let total = text.chars().count(); + if cap >= total { + self.remaining = self.remaining.saturating_sub(total); + return (text.to_string(), 0); } + let taken: String = text.chars().take(cap).collect(); + self.remaining = self.remaining.saturating_sub(cap); + (taken, total - cap) + } +} + +/// A tool call the assistant chose, as recovered from a `tool_use` block. +#[derive(Debug, Clone)] +struct ToolCall { + id: String, + name: String, + arguments: Option, +} + +/// The observation a tool call produced, as recovered from a `tool_result` block. +#[derive(Debug, Clone, Default)] +struct Observation { + is_error: bool, + text: String, +} + +fn tool_calls_from_entry(entry: &TreeEntry) -> Vec { + let Some(blocks) = entry.blocks() else { + return Vec::new(); + }; + blocks + .iter() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) + .filter_map(|block| { + let id = block.get("id").and_then(Value::as_str)?.to_string(); + Some(ToolCall { + name: block + .get("name") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(), + arguments: block.get("input").cloned(), + id, + }) + }) + .collect() +} + +fn observations_from_entry(entry: &TreeEntry) -> Vec<(String, Observation)> { + let Some(blocks) = entry.blocks() else { + return Vec::new(); + }; + blocks + .iter() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result")) + .filter_map(|block| { + let id = block.get("tool_use_id").and_then(Value::as_str)?.to_string(); + let text = match block.get("content") { + Some(Value::String(s)) => s.clone(), + Some(other) => serde_json::to_string(other).unwrap_or_default(), + None => String::new(), + }; + Some(( + id, + Observation { + is_error: block + .get("is_error") + .and_then(Value::as_bool) + .unwrap_or(false), + text, + }, + )) + }) + .collect() +} + +/// Bound a tool-call argument payload, charging it to the trajectory's inline +/// budget. Arguments are the largest attacker-shaped field in a decision — a +/// file-write tool call carries the whole file — so they are capped per call +/// and counted against the same global ceiling as message text. +fn bound_arguments(arguments: Option, budget: &mut InlineBudget) -> Option { + let arguments = arguments?; + if arguments.is_null() { + return None; + } + let serialized = serde_json::to_string(&arguments).unwrap_or_default(); + let (preview, dropped) = budget.take(&serialized, MAX_ARGUMENTS_CHARS); + if dropped == 0 { + return Some(arguments); + } + Some(json!({ + "_truncated": true, + "_original_chars": serialized.chars().count(), + "_preview": preview, + })) +} + +fn parse_arguments_field(value: Option<&Value>) -> Option { + match value { + Some(Value::String(s)) => Some( + serde_json::from_str::(s).unwrap_or_else(|_| Value::String(s.clone())), + ), + Some(Value::Null) | None => None, + Some(other) => Some(other.clone()), + } +} + +/// Build a decision from a tool call plus whatever evidence exists for it. +fn build_decision( + call: &ToolCall, + observation: Option<&Observation>, + span: Option<&Value>, + budget: &mut InlineBudget, +) -> Value { + let name = if call.name.is_empty() || call.name == "unknown" { + span.and_then(|s| s.get("tool_name")) + .and_then(Value::as_str) + .unwrap_or(&call.name) + .to_string() + } else { + call.name.clone() + }; + + let arguments = call + .arguments + .clone() + .filter(|value| !value.is_null()) + .or_else(|| parse_arguments_field(span.and_then(|s| s.get("arguments")))); + + let mut choice = json!({ "action": if name.is_empty() { "unknown".to_string() } else { name } }); + if let Some(arguments) = bound_arguments(arguments, budget) { + choice["arguments"] = arguments; } - let truncated_result = truncate_chars(result, 500); + let (is_error, result_text) = match observation { + Some(observation) => (observation.is_error, observation.text.clone()), + None => ( + span.and_then(|s| s.get("is_error")) + .and_then(Value::as_bool) + .unwrap_or(false), + span.and_then(|s| s.get("result")) + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + ), + }; + let mut consequence = json!({ "success": !is_error, - "result_summary": truncated_result, + "result_summary": truncate_chars(&result_text, MAX_RESULT_SUMMARY_CHARS), }); if is_error { - consequence["error_type"] = json!(classify_error(result)); + consequence["error_type"] = json!(classify_error(&result_text)); } - json!({ - "decision_id": tool_call_id, + let mut decision = json!({ + "decision_id": call.id, "decision_type": "tool_selection", + // cause_id links the decision to the observation it caused: the + // tool_result block carrying the same tool_call_id, which lands in the + // next turn's prompt side. + "cause_id": call.id, "choice": choice, "consequence": consequence, - "_duration_ms": duration_ms, - }) + }); + if let Some(duration) = span.and_then(|s| s.get("duration_ms")).and_then(json_u64) { + decision["_duration_ms"] = json!(duration); + } + decision +} + +/// Convert a single tool-span JSON object (as emitted by +/// `monty_repl::emit_tool_call_telemetry`) into an `OTSDecision` JSON value. +/// +/// Used for spans that no assistant entry claims — the session tree body was +/// externalized to TemperFS, so the span is the only surviving evidence of the +/// call. `_duration_ms` is preserved as a non-standard field for the evaluation +/// agents; the OTS schema has no home for tool wall-clock time. +fn span_to_decision(span: &Value, budget: &mut InlineBudget) -> Value { + let call = ToolCall { + id: span + .get("tool_call_id") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(), + name: span + .get("tool_name") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(), + arguments: None, + }; + build_decision(&call, None, Some(span), budget) +} + +/// Reserved tool name `monty_repl` writes when the span document hits its size +/// ceiling. It marks an incomplete record, not a decision the agent made. +pub const TOOL_SPANS_TRUNCATED_MARKER: &str = "_tool_spans_truncated"; + +fn is_truncation_marker(span: &Value) -> bool { + span.get("tool_name").and_then(Value::as_str) == Some(TOOL_SPANS_TRUNCATED_MARKER) +} + +/// A parsed tool-span document. +pub struct ToolSpanDocument { + /// Real spans, in execution order. + pub spans: Vec, + /// True when the document was sealed at its size ceiling, so the decision + /// record for this session is knowingly incomplete. + pub truncated: bool, + /// Lines that did not parse. Skipping them keeps one bad append from + /// costing every span, but they are still tool calls with no evidence left, + /// so the count travels rather than the loss being silent. + pub unparsed_lines: usize, } -/// Parse a tool-span JSONL document into a vector of `OTSDecision` JSON values. +/// Parse a tool-span JSONL document into span values, preserving execution +/// order and separating out the truncation marker. /// /// Invalid lines are skipped silently — tool-span persistence is best-effort and -/// the emitter must not fail on a corrupted span. -pub fn parse_tool_spans_to_decisions(tool_spans_jsonl: &str) -> Vec { - tool_spans_jsonl - .lines() - .filter(|line| !line.trim().is_empty()) - .filter_map(|line| serde_json::from_str::(line).ok()) - .map(|span| span_to_decision(&span)) - .collect() +/// the emitter must not fail on a corrupted span. Truncation is decided from the +/// reserved `tool_name` of a parsed record, never from a substring search: a +/// tool that reads or greps the emitter's own source returns the marker literal +/// in its result, and that must not make a complete run look partial. +pub fn parse_tool_span_document(tool_spans_jsonl: &str) -> ToolSpanDocument { + let mut spans = Vec::new(); + let mut truncated = false; + let mut unparsed_lines = 0; + for line in tool_spans_jsonl.lines().map(str::trim) { + if line.is_empty() { + continue; + } + let Ok(span) = serde_json::from_str::(line) else { + unparsed_lines += 1; + continue; + }; + if is_truncation_marker(&span) { + truncated = true; + } else { + spans.push(span); + } + } + ToolSpanDocument { + spans, + truncated, + unparsed_lines, + } } /// Extract first and last event timestamps from the entity event log. @@ -116,7 +687,6 @@ pub fn parse_tool_spans_to_decisions(tool_spans_jsonl: &str) -> Vec { /// when no events exist — keeps the schema populated with a legal value rather /// than failing deserialization. pub fn extract_event_bookends(entity_state: &Value) -> (String, String) { - const EPOCH: &str = "1970-01-01T00:00:00Z"; let events = entity_state.get("events").and_then(|v| v.as_array()); let Some(events) = events else { return (EPOCH.to_string(), EPOCH.to_string()); @@ -136,395 +706,2883 @@ pub fn extract_event_bookends(entity_state: &Value) -> (String, String) { (first, last) } -/// Assemble a complete `OTSTrajectory` JSON document. +/// Timestamps of the events that close an LLM cycle, oldest first. /// -/// Decisions are collapsed into a single synthetic `OTSTurn` for MVP. Precise -/// turn-boundary reconstruction from the session tree is deferred to a follow-up -/// track — see ADR-0035 risks section 1. -pub fn build_trajectory( - trajectory_id: &str, - session_id: &str, - agent_id: &str, - status: &str, - fields: &Value, - tool_spans_jsonl: &str, - entity_state: &Value, -) -> Value { - let (timestamp_start, timestamp_end) = extract_event_bookends(entity_state); - let has_result = fields - .get("has_result") - .and_then(|v| v.as_bool()) - .or_else(|| { - fields - .get("has_result") - .and_then(|v| v.as_str()) - .map(|s| s == "true") +/// The entity event log is a hot tail (older events are dropped at snapshot +/// boundaries), so this is a fallback for entries written before per-entry +/// timestamps were recorded — never the primary source. +fn turn_boundary_event_timestamps(entity_state: &Value) -> Vec { + const CYCLE_CLOSING_ACTIONS: &[&str] = &[ + "ProcessToolCalls", + "CheckSteering", + "RecordResult", + "RecordResultNoReply", + "RecordResultInlineReply", + ]; + entity_state + .get("events") + .and_then(Value::as_array) + .map(|events| { + events + .iter() + .filter(|event| { + event + .get("action") + .and_then(Value::as_str) + .is_some_and(|action| CYCLE_CLOSING_ACTIONS.contains(&action)) + }) + .filter_map(|event| event.get("timestamp").and_then(Value::as_str)) + .map(str::to_string) + .collect() }) - .unwrap_or(false); - let outcome = derive_outcome(status, has_result); + .unwrap_or_default() +} - let task_description = truncate_chars( - fields - .get("user_message") - .and_then(|v| v.as_str()) - .unwrap_or(""), - 500, - ); - let model = fields - .get("model") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let provider = fields - .get("provider") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let session_mode = fields - .get("session_mode") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); +fn field_str<'a>(fields: &'a Value, key: &str) -> &'a str { + fields.get(key).and_then(Value::as_str).unwrap_or("") +} - let mut tags: Vec = Vec::new(); - if !model.is_empty() { - tags.push(model); +fn message_role(entry: &TreeEntry) -> &'static str { + if entry.is_assistant() { + return "assistant"; } - if !provider.is_empty() { - tags.push(provider); + if entry.entry_type == "compaction" { + return "system"; } - if !session_mode.is_empty() { - tags.push(session_mode); + let has_tool_results = entry + .blocks() + .is_some_and(|blocks| { + blocks + .iter() + .any(|block| block.get("type").and_then(Value::as_str) == Some("tool_result")) + }); + if has_tool_results || entry.id.starts_with("t-") { + return "tool"; } + "user" +} - let decisions = parse_tool_spans_to_decisions(tool_spans_jsonl); - - // OTSTurn requires turn_id (i32), span_id (String), and timestamp (RFC-3339). - // We emit exactly one synthetic turn per session for the MVP emitter; follow-up - // tracks will reconstruct real per-LLM-cycle boundaries from the session tree. - let turn = json!({ - "turn_id": 1, - "span_id": session_id, - "timestamp": timestamp_start, - "decisions": decisions, - }); - - // trajectory_id is duplicated inside `metadata` because Temper's server-side - // POST handler at temper-server/src/observe/evolution/trajectories.rs reads - // it from metadata.trajectory_id (not the OTS top-level field). Emitting in - // both places keeps OTS schema compliance AND lets the Turso row use the same - // id my module stored on the Session entity — which is what makes - // INSERT OR REPLACE-based retry idempotency actually work. - // - // timestamp_start and timestamp_end are required by the OTSMetadata struct - // at temper/crates/temper-ots/src/models/metadata.rs; omitting them breaks - // deserialization even though the blob persists as opaque text. - let metadata = json!({ - "trajectory_id": trajectory_id, - "task_description": task_description, - "domain": "temperpaw-agent", - "timestamp_start": timestamp_start, - "timestamp_end": timestamp_end, - "agent_id": agent_id, - "framework": "temperpaw", - "environment": "production", - "outcome": outcome, - "tags": tags, - }); - - let mut trajectory = json!({ - "trajectory_id": trajectory_id, - "version": "0.1.0", - "metadata": metadata, - "turns": [turn], +/// Build the OTS message for one session entry, honoring the inline budget. +fn build_message(entry: &TreeEntry, timestamp: &str, budget: &mut InlineBudget) -> Value { + let role = message_role(entry); + let mut message = json!({ + "message_id": entry.id, + "role": role, + "timestamp": entry.recorded_at().unwrap_or_else(|| timestamp.to_string()), }); - let system_prompt = fields - .get("system_prompt") - .and_then(|v| v.as_str()) - .unwrap_or(""); - if !system_prompt.is_empty() { - // OTSSystemMessage only has { content, timestamp } — no `role` field. - trajectory["system_message"] = json!({ - "content": truncate_chars(system_prompt, 2000), - "timestamp": timestamp_start, + // Externalized bodies are referenced, never fetched and never inlined. + if let Some(file_id) = &entry.content_file_id { + let mut data = json!({ + "content_file_id": file_id, + "externalized": true, + "tokens": entry.tokens, + }); + if let Some(version_id) = &entry.content_file_version_id { + data["content_file_version_id"] = json!(version_id); + } + message["content"] = json!({ + "type": content_type_for_role(role), + "data": data, }); + return message; } - trajectory -} + let mut content = json!({ "type": "text" }); + let mut data = Map::new(); -#[cfg(test)] -mod tests { - use super::*; + match entry.content.clone() { + Some(Value::String(text)) => { + let (text, dropped) = budget.take(&text, MAX_MESSAGE_INLINE_CHARS); + content["text"] = json!(text); + if dropped > 0 { + data.insert("truncated_chars".to_string(), json!(dropped)); + } + } + Some(Value::Array(blocks)) => { + let mut text_parts: Vec = Vec::new(); + let mut reasoning_parts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + let mut tool_results: Vec = Vec::new(); - #[test] - fn derive_outcome_matrix() { - assert_eq!(derive_outcome("Completed", true), "success"); - assert_eq!(derive_outcome("Completed", false), "partial_success"); - assert_eq!(derive_outcome("Failed", true), "failure"); - assert_eq!(derive_outcome("Failed", false), "failure"); - assert_eq!(derive_outcome("Cancelled", true), "partial_success"); - assert_eq!(derive_outcome("Cancelled", false), "partial_success"); - assert_eq!(derive_outcome("Thinking", true), "failure"); // defensive - } + for block in &blocks { + match block.get("type").and_then(Value::as_str).unwrap_or("") { + "text" => { + if let Some(text) = block.get("text").and_then(Value::as_str) { + text_parts.push(text.to_string()); + } + } + "thinking" | "redacted_thinking" => { + if let Some(text) = block + .get("thinking") + .or_else(|| block.get("text")) + .and_then(Value::as_str) + { + reasoning_parts.push(text.to_string()); + } + } + "tool_use" => { + // Identity only. The arguments live on the decision for + // this call; duplicating them here would double the + // largest field in the document for no new information. + tool_calls.push(json!({ + "id": block.get("id").and_then(Value::as_str).unwrap_or(""), + "name": block.get("name").and_then(Value::as_str).unwrap_or(""), + })); + } + "tool_result" => { + let text = match block.get("content") { + Some(Value::String(s)) => s.clone(), + Some(other) => serde_json::to_string(other).unwrap_or_default(), + None => String::new(), + }; + let (text, dropped) = budget.take(&text, MAX_MESSAGE_INLINE_CHARS); + let mut result = json!({ + "tool_call_id": block + .get("tool_use_id") + .and_then(Value::as_str) + .unwrap_or(""), + "is_error": block + .get("is_error") + .and_then(Value::as_bool) + .unwrap_or(false), + "content": text, + }); + if dropped > 0 { + result["truncated_chars"] = json!(dropped); + } + tool_results.push(result); + } + _ => {} + } + } - #[test] - fn classify_error_categories() { - assert_eq!(classify_error(""), "unknown_error"); - assert_eq!(classify_error("request timed out after 30s"), "tool_timeout"); - assert_eq!(classify_error("Cedar denied action"), "cedar_denied"); - assert_eq!( - classify_error("permission denied by policy"), - "cedar_denied" - ); - assert_eq!(classify_error("subprocess failed"), "tool_error"); + if !text_parts.is_empty() { + let (text, dropped) = budget.take(&text_parts.join("\n"), MAX_MESSAGE_INLINE_CHARS); + content["text"] = json!(text); + if dropped > 0 { + data.insert("truncated_chars".to_string(), json!(dropped)); + } + } + if !reasoning_parts.is_empty() { + let (reasoning, _) = + budget.take(&reasoning_parts.join("\n"), MAX_MESSAGE_INLINE_CHARS); + message["reasoning"] = json!(reasoning); + } + if !tool_calls.is_empty() { + content["type"] = json!("tool_call"); + data.insert("tool_calls".to_string(), json!(tool_calls)); + } + if !tool_results.is_empty() { + content["type"] = json!("tool_response"); + data.insert("tool_results".to_string(), json!(tool_results)); + } + } + Some(other) if !other.is_null() => { + let serialized = serde_json::to_string(&other).unwrap_or_default(); + let (text, dropped) = budget.take(&serialized, MAX_MESSAGE_INLINE_CHARS); + content["text"] = json!(text); + if dropped > 0 { + data.insert("truncated_chars".to_string(), json!(dropped)); + } + } + _ => {} } - #[test] - fn truncate_chars_respects_utf8() { - let s = "héllo wörld"; - assert_eq!(truncate_chars(s, 5), "héllo"); - assert_eq!(truncate_chars(s, 100), s); + // Compaction entries keep their summary in an extra field, not `content`. + if entry.entry_type == "compaction" + && let Some(summary) = entry.raw.get("summary").and_then(Value::as_str) + { + let (text, dropped) = budget.take(summary, MAX_MESSAGE_INLINE_CHARS); + content["text"] = json!(text); + data.insert("compaction".to_string(), json!(true)); + if dropped > 0 { + data.insert("truncated_chars".to_string(), json!(dropped)); + } } - #[test] - fn span_to_decision_success_shape() { - let span = json!({ - "tool_name": "temper.read", - "tool_call_id": "tc-123", - "arguments": "{\"path\":\"/tmp/foo\"}", - "result": "ok", - "duration_ms": 42, - "is_error": false, - }); - let dec = span_to_decision(&span); - assert_eq!(dec["decision_id"], "tc-123"); - assert_eq!(dec["decision_type"], "tool_selection"); - assert_eq!(dec["choice"]["action"], "temper.read"); - assert_eq!(dec["choice"]["arguments"]["path"], "/tmp/foo"); - assert_eq!(dec["consequence"]["success"], true); - assert_eq!(dec["consequence"]["result_summary"], "ok"); - assert!(dec["consequence"].get("error_type").is_none()); - assert_eq!(dec["_duration_ms"], 42); + if !data.is_empty() { + content["data"] = Value::Object(data); } + message["content"] = content; + message +} - #[test] - fn span_to_decision_error_classifies() { - let span = json!({ - "tool_name": "temper.bash", - "tool_call_id": "tc-err", - "arguments": "{}", - "result": "Cedar denied bash execution", - "duration_ms": 5, - "is_error": true, - }); - let dec = span_to_decision(&span); - assert_eq!(dec["consequence"]["success"], false); - assert_eq!(dec["consequence"]["error_type"], "cedar_denied"); +fn content_type_for_role(role: &str) -> &'static str { + match role { + "tool" => "tool_response", + _ => "text", } +} +/// Shape check for a token-signal array before it is written to a turn. +type SignalValidator = fn(&Value) -> bool; + +/// Completion-side token signals, in the order they are emitted. Element *i* of +/// each one describes the same generated token, so they are positionally +/// aligned with one another. +const COMPLETION_TOKEN_SIGNALS: &[(&str, SignalValidator)] = &[ + ("completion_token_ids", is_u32_array), + ("response_mask", is_mask_array), + ("logprobs", is_f64_array), +]; + +/// The completion-side signal the other two index into. The kernel requires it +/// to be present whenever they are, so the emitter does too. +const COMPLETION_ANCHOR: &str = "completion_token_ids"; + +/// Whole-trajectory ceiling on token-signal bytes, spent in write order. +struct TokenSignalBudget { + remaining: usize, +} + +impl TokenSignalBudget { + fn new(total: usize) -> Self { + TokenSignalBudget { remaining: total } + } + + /// Serialized size of `value`, or `None` when it does not fit what is left. + fn take(&mut self, value: &Value) -> Option { + let size = serde_json::to_string(value).ok()?.len(); + if size > self.remaining { + return None; + } + self.remaining -= size; + Some(size) + } +} + +/// Copy token-id / mask / logprob signals onto the turn when the serving stack +/// recorded them. Absent otherwise — the emitter never fabricates them and never +/// makes a provider round-trip to fetch them. +/// +/// The completion-side signals are emitted only when the ones that are present +/// agree on length. Arrays of different lengths would hand an RL consumer +/// probabilities and mask bits belonging to the wrong tokens, which is worse +/// than having none: the misalignment is invisible downstream. A dropped set is +/// recorded as `_token_signals_misaligned` so the loss is not silent, and a set +/// dropped for exceeding `MAX_TOKEN_SIGNAL_BYTES` as `_token_signals_dropped`. +/// +/// Returns an inventory of what was written: signal name -> element count, plus +/// any drop marker. The caller reads it to tell whether the turn's record came +/// up short. +fn attach_token_signals( + turn: &mut Value, + source: &Value, + budget: &mut TokenSignalBudget, +) -> Map { + let mut inventory = Map::new(); + + // Signals the SessionEntry writer already refused, before the emitter ever + // saw them. Without carrying these forward, a turn whose signals were all + // dropped at capture is indistinguishable from a provider that sent none. + for field in TOKEN_SIGNAL_FIELDS { + let marker = format!("{field}_dropped_bytes"); + if let Some(size) = source.get(&marker).and_then(json_u64) { + record_signal_drop_size(turn, &mut inventory, field, size); + } + } + + // The entry's extras were themselves cut to fit the field ceiling. Which + // members went is no longer knowable — only how many — but the turn's + // record is short either way. + if let Some(dropped) = source.get("_extra_json_dropped_members").and_then(json_u64) + && dropped > 0 + { + turn["_turn_extras_dropped_members"] = json!(dropped); + inventory.insert("_turn_extras_dropped_members".to_string(), json!(dropped)); + } + + // Prompt-side ids describe the prompt, which the completion signals do not + // index into, so they stand on their own. + if let Some(value) = source + .get("prompt_token_ids") + .filter(|value| is_u32_array(value)) + { + if budget.take(value).is_some() { + inventory.insert( + "prompt_token_ids".to_string(), + json!(value.as_array().map(Vec::len).unwrap_or(0)), + ); + turn["prompt_token_ids"] = value.clone(); + } else { + record_signal_drop(turn, &mut inventory, "prompt_token_ids", value); + } + } + + let present: Vec<(&str, &Value)> = COMPLETION_TOKEN_SIGNALS + .iter() + .filter_map(|(field, validator)| { + source + .get(*field) + .filter(|value| validator(value)) + .map(|value| (*field, value)) + }) + .collect(); + if present.is_empty() { + return inventory; + } + + // `completion_token_ids` is the anchor the other two index into, and the + // kernel enforces exactly that: `OTSTurn::validate_token_signals` rejects a + // turn carrying `response_mask` or `logprobs` without it. A rejected + // document is not a partial write — the POST answers 400, the emission is + // recorded failed, and `build_trajectory` is deterministic, so every retry + // rebuilds the identical rejected document and the row is lost for good. + // + // Anchorless sets reach here for real rather than in theory: the + // SessionEntry writer bounds each signal independently, and the ids are + // several times the size of the mask, so a long completion loses the ids + // while the mask survives. + if !present.iter().any(|(field, _)| *field == COMPLETION_ANCHOR) { + for (field, value) in present { + record_signal_drop(turn, &mut inventory, field, value); + } + return inventory; + } + + let mut lengths = Map::new(); + for (field, value) in &present { + lengths.insert( + (*field).to_string(), + json!(value.as_array().map(Vec::len).unwrap_or(0)), + ); + } + let aligned = lengths + .values() + .map(|length| length.as_u64().unwrap_or(0)) + .collect::>() + .len() + <= 1; + + if !aligned { + turn["_token_signals_misaligned"] = Value::Object(lengths.clone()); + inventory.insert("_token_signals_misaligned".to_string(), Value::Object(lengths)); + return inventory; + } + + // The aligned set travels or is dropped whole: keeping a mask without the + // ids it indexes leaves a consumer with signal it cannot use. + let set = Value::Array(present.iter().map(|(_, value)| (*value).clone()).collect()); + if budget.take(&set).is_none() { + for (field, value) in present { + record_signal_drop(turn, &mut inventory, field, value); + } + return inventory; + } + for (field, value) in present { + inventory.insert( + field.to_string(), + json!(value.as_array().map(Vec::len).unwrap_or(0)), + ); + turn[field] = value.clone(); + } + inventory +} + +/// Record a signal the trajectory budget refused, by element count. A dropped +/// signal that leaves a trace is debuggable; a silent one reads as a turn the +/// serving stack never produced signals for. +fn record_signal_drop( + turn: &mut Value, + inventory: &mut Map, + field: &str, + value: &Value, +) { + record_signal_drop_size( + turn, + inventory, + field, + value.as_array().map(Vec::len).unwrap_or(0) as u64, + ); +} + +/// Same record, from a size the caller already knows — the SessionEntry writer +/// reports bytes rather than elements when it refuses a signal at capture. +fn record_signal_drop_size( + turn: &mut Value, + inventory: &mut Map, + field: &str, + size: u64, +) { + let entry = json!(size); + match turn + .get_mut("_token_signals_dropped") + .and_then(Value::as_object_mut) + { + Some(existing) => { + existing.insert(field.to_string(), entry.clone()); + } + None => { + let mut map = Map::new(); + map.insert(field.to_string(), entry.clone()); + turn["_token_signals_dropped"] = Value::Object(map); + } + } + let carried = inventory + .entry("_token_signals_dropped".to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if let Some(map) = carried.as_object_mut() { + map.insert(field.to_string(), entry); + } +} + +fn is_u32_array(value: &Value) -> bool { + value + .as_array() + .is_some_and(|items| items.iter().all(|item| item.as_u64().is_some_and(|n| n <= u32::MAX as u64))) +} + +/// `response_mask` is a per-token loss switch: every entry is 0 or 1. The kernel +/// rejects the whole turn over a single entry above 1, so a looser check here +/// would hand it a document it refuses — and a refused document is lost, not +/// degraded (`validate_token_signals`). +fn is_mask_array(value: &Value) -> bool { + value + .as_array() + .is_some_and(|items| items.iter().all(|item| matches!(item.as_u64(), Some(0 | 1)))) +} + +fn is_f64_array(value: &Value) -> bool { + value + .as_array() + .is_some_and(|items| items.iter().all(|item| item.as_f64().is_some())) +} + +/// Take the earliest span carrying `id` that no decision has claimed yet. +/// +/// Repeated ids are matched in execution order, so the first `tool_1` call gets +/// the first `tool_1` span. Without this, a document-wide `id -> span` map hands +/// every call sharing an id the same duration and result. +fn claim_span( + span_queue_by_id: &BTreeMap<&str, Vec>, + span_claimed: &mut [bool], + id: &str, +) -> Option { + let queue = span_queue_by_id.get(id)?; + let index = *queue.iter().find(|index| !span_claimed[**index])?; + span_claimed[index] = true; + Some(index) +} + +/// Prefix marking a `session_file_id` that points at SessionEntry rows rather +/// than a TemperFS file. Mirrors `wasm_helpers::session_entries_ref`. +const SESSION_ENTRIES_REF_PREFIX: &str = "session-entries:"; + +fn file_resource(resources: &mut Vec, kind: &str, file_id: &str) { + if file_id.is_empty() { + return; + } + // The session tree is entity-backed in production, so its "file id" is a + // SessionEntries reference. Pointing a consumer at Files('session-entries:…') + // would send it somewhere that does not exist. + let uri = match file_id.strip_prefix(SESSION_ENTRIES_REF_PREFIX) { + Some(session_id) => format!("temper://SessionEntries?SessionId={session_id}"), + None => format!("temperfs://Files('{file_id}')"), + }; + resources.push(json!({ "type": kind, "uri": uri })); +} + +/// Evidence the finished document was built without, newest-first in the order +/// the tags were added. Empty for a complete trajectory. +/// +/// Read back off the document rather than recomputed from the inputs, so the +/// stored row and the Session entity cannot disagree about what is missing. +pub fn degradations(trajectory: &Value) -> Vec { + trajectory["metadata"]["tags"] + .as_array() + .map(|tags| { + tags.iter() + .filter_map(Value::as_str) + .filter_map(|tag| tag.strip_prefix(DEGRADED_TAG_PREFIX)) + .map(str::to_string) + .collect() + }) + .unwrap_or_default() +} + +/// Assemble a complete `OTSTrajectory` JSON document. +pub fn build_trajectory(inputs: &TrajectoryInputs<'_>) -> Value { + let TrajectoryInputs { + trajectory_id, + session_id, + agent_id, + status, + fields, + session_jsonl, + tool_spans_jsonl, + entity_state, + spec_version, + transcript, + tool_spans_missing, + } = *inputs; + + let (event_start, timestamp_end) = extract_event_bookends(entity_state); + let entries = parse_session_entries(session_jsonl); + // A transcript that is there but does not parse is missing history just as + // surely as one that is absent, and `parse_session_entries` drops bad lines + // deliberately so a single corrupted line cannot cost the whole trajectory. + // Judging completeness by whether bytes arrived would let a truncated or + // corrupted file store as a complete record. + let unparsed_lines = session_jsonl + .lines() + .filter(|line| !line.trim().is_empty()) + .count() + .saturating_sub(entries.len()); + let resolved = resolve_chain(&entries, field_str(fields, "session_leaf_id")); + let chain = resolved.chain; + let turn_drafts = group_turns(&entries, &chain); + + let timestamp_start = chain + .iter() + .find_map(|index| entries[*index].recorded_at()) + .unwrap_or(event_start); + + let has_result = fields + .get("has_result") + .and_then(|v| v.as_bool()) + .or_else(|| { + fields + .get("has_result") + .and_then(|v| v.as_str()) + .map(|s| s == "true") + }) + .unwrap_or(false); + let outcome = derive_outcome(status, has_result); + + let task_description = truncate_chars( + field_str(fields, "user_message"), + MAX_TASK_DESCRIPTION_CHARS, + ); + + let mut tags: Vec = Vec::new(); + for key in ["model", "provider", "session_mode"] { + let value = field_str(fields, key); + if !value.is_empty() { + tags.push(value.to_string()); + } + } + // What the document was built without. `metadata.tags` is the carrier + // because a consumer that loses a completeness marker reads a partial + // record as a whole one — and unlike run provenance, which the kernel now + // models directly, there is no field of its own for this. + // + // Absence is only the loudest way a transcript can be short. It can also + // arrive corrupted, arrive without the newest turns (the recorded leaf does + // not resolve, so the fallback chain is an older one), or arrive with + // entries that yield no turn at all. Each of those produces a document that + // reads as a smaller session than the one that ran, and each is reported. + // + // `turn_count` is deliberately not one of these. It counts continuations + // (tool results, steering, plan resumes), not assistant messages, so it + // does not equal the reconstructed turn count even on a healthy session — + // comparing them would mark almost every trajectory degraded. It travels as + // `_session_turn_count` for a consumer that wants to weigh the two. + let mut transcript_reasons: Vec<&str> = Vec::new(); + if !transcript.is_present() { + transcript_reasons.push(transcript.as_str()); + } else { + if unparsed_lines > 0 { + transcript_reasons.push("unparseable"); + } + if !resolved.from_recorded_leaf { + transcript_reasons.push("leaf_unresolved"); + } + if turn_drafts.is_empty() { + transcript_reasons.push("no_turns"); + } + } + for reason in &transcript_reasons { + tags.push(format!("{DEGRADED_TAG_PREFIX}transcript_{reason}")); + } + let span_document = parse_tool_span_document(tool_spans_jsonl); + if span_document.truncated { + tags.push(format!("{DEGRADED_TAG_PREFIX}tool_spans_truncated")); + } + if span_document.unparsed_lines > 0 { + tags.push(format!("{DEGRADED_TAG_PREFIX}tool_spans_unparseable")); + } + if tool_spans_missing { + tags.push(format!("{DEGRADED_TAG_PREFIX}tool_spans_missing_file")); + } + // The session recorded that an append failed. Its tool calls happened and + // their record did not survive, which an empty span file cannot distinguish + // from a session that called nothing. + let tool_spans_write_failed = field_str(fields, "tool_spans_write_failed") == "true"; + if tool_spans_write_failed { + tags.push(format!("{DEGRADED_TAG_PREFIX}tool_spans_write_failed")); + } + + // Tool-call ids are unique only within a turn. Providers that omit them get + // synthetic ids that restart with each response, and a model can repeat one, + // so a document-wide index would let a later turn's call overwrite an + // earlier one — both decisions would then carry the last call's consequence + // and duration. Observations and spans are therefore attributed per turn. + // + // A tool_result on turn N's prompt answers a call made in turn N-1, so the + // observations parsed from turn K's prompt belong to turn K-1. Ones landing + // on the first turn answer a call made before this chain begins and have no + // decision to attach to. + let mut observations_by_turn: Vec> = + vec![Vec::new(); turn_drafts.len()]; + for (turn_index, draft) in turn_drafts.iter().enumerate() { + let Some(answered_turn) = turn_index.checked_sub(1) else { + continue; + }; + for prompt_index in &draft.prompt { + observations_by_turn[answered_turn] + .extend(observations_from_entry(&entries[*prompt_index])); + } + } + + let spans = span_document.spans; + // Spans are appended in execution order, so repeated ids are matched to + // calls first-come-first-served rather than collapsed onto one record. + let mut span_queue_by_id: BTreeMap<&str, Vec> = BTreeMap::new(); + for (index, span) in spans.iter().enumerate() { + if let Some(id) = span.get("tool_call_id").and_then(Value::as_str) { + span_queue_by_id.entry(id).or_default().push(index); + } + } + let mut span_claimed: Vec = vec![false; spans.len()]; + + let boundary_timestamps = turn_boundary_event_timestamps(entity_state); + let mut budget = InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS); + let mut signal_budget = TokenSignalBudget::new(MAX_TOKEN_SIGNAL_BYTES); + let mut lost_token_signals = false; + let mut turns: Vec = Vec::new(); + + for (turn_index, draft) in turn_drafts.iter().enumerate() { + let assistant = draft.assistant.map(|index| &entries[index]); + let timestamp = assistant + .and_then(|entry| entry.recorded_at()) + .or_else(|| boundary_timestamps.get(turn_index).cloned()) + .unwrap_or_else(|| timestamp_start.clone()); + + let mut messages: Vec = Vec::new(); + for prompt_index in &draft.prompt { + messages.push(build_message(&entries[*prompt_index], ×tamp, &mut budget)); + } + if let Some(entry) = assistant { + messages.push(build_message(entry, ×tamp, &mut budget)); + } + + // Decisions the assistant made in this cycle, in call order. Everything + // below is scoped to this turn, because ids repeat across turns. + let mut calls: Vec = Vec::new(); + let mut named: BTreeSet = BTreeSet::new(); + if let Some(entry) = assistant { + for call in tool_calls_from_entry(entry) { + named.insert(call.id.clone()); + calls.push(call); + } + } + // Calls whose only evidence is the observation — the assistant body was + // externalized, so the tree cannot name them — still deserve a decision. + // `build_decision` recovers the tool name from the span when one exists. + for (id, _) in &observations_by_turn[turn_index] { + if named.insert(id.clone()) { + calls.push(ToolCall { + id: id.clone(), + name: "unknown".to_string(), + arguments: None, + }); + } + } + + let mut observation_by_id: BTreeMap<&str, &Observation> = BTreeMap::new(); + for (id, observation) in &observations_by_turn[turn_index] { + observation_by_id.entry(id.as_str()).or_insert(observation); + } + + let mut decisions: Vec = Vec::new(); + let mut turn_error = false; + let mut turn_duration_ms: u64 = 0; + for call in &calls { + let span_index = claim_span(&span_queue_by_id, &mut span_claimed, &call.id); + let decision = build_decision( + call, + observation_by_id.get(call.id.as_str()).copied(), + span_index.map(|index| &spans[index]), + &mut budget, + ); + if decision["consequence"]["success"] == json!(false) { + turn_error = true; + } + if let Some(duration) = decision.get("_duration_ms").and_then(json_u64) { + turn_duration_ms += duration; + } + decisions.push(decision); + } + + let span_id = assistant + .map(|entry| format!("{session_id}:{}", entry.id)) + .unwrap_or_else(|| format!("{session_id}:turn-{}", turn_index + 1)); + + let mut turn = json!({ + "turn_id": (turn_index + 1) as i64, + "span_id": span_id, + "timestamp": timestamp, + "error": turn_error, + "messages": messages, + "decisions": decisions, + }); + if turn_duration_ms > 0 { + turn["duration_ms"] = json!(turn_duration_ms as f64); + } + + let prompt_tokens: u64 = assistant + .and_then(|entry| entry.raw.get("input_tokens").and_then(json_u64)) + .unwrap_or_else(|| { + draft + .prompt + .iter() + .map(|index| entries[*index].tokens) + .sum() + }); + let completion_tokens: u64 = assistant + .and_then(|entry| { + entry + .raw + .get("output_tokens") + .and_then(json_u64) + .or(Some(entry.tokens)) + }) + .unwrap_or(0); + turn["_prompt_tokens"] = json!(prompt_tokens); + turn["_completion_tokens"] = json!(completion_tokens); + + if let Some(entry) = assistant { + let inventory = attach_token_signals(&mut turn, &entry.raw, &mut signal_budget); + // Misalignment discards the completion-side set exactly as a budget + // drop does, and extras cut to fit the entry ceiling take turn facts + // with them. All three are the record coming up short. + lost_token_signals |= inventory.contains_key("_token_signals_dropped") + || inventory.contains_key("_token_signals_misaligned") + || inventory.contains_key("_turn_extras_dropped_members"); + } + + turns.push(turn); + } + + // Spans no turn claimed (whole tree unavailable, or a call the tree never + // recorded) still carry real decisions — attach them to the last turn so no + // evidence is silently dropped. Claiming is tracked by span position, so two + // spans sharing an id are two decisions, not one. + let orphan_decisions: Vec = spans + .iter() + .enumerate() + .filter(|(index, _)| !span_claimed[*index]) + .map(|(_, span)| span_to_decision(span, &mut budget)) + .collect(); + + if !orphan_decisions.is_empty() { + if turns.is_empty() { + turns.push(json!({ + "turn_id": 1_i64, + "span_id": format!("{session_id}:turn-1"), + "timestamp": timestamp_start, + "error": orphan_decisions + .iter() + .any(|d| d["consequence"]["success"] == json!(false)), + "messages": Vec::::new(), + "decisions": orphan_decisions, + })); + } else { + let last = turns.len() - 1; + if orphan_decisions + .iter() + .any(|d| d["consequence"]["success"] == json!(false)) + { + turns[last]["error"] = json!(true); + } + if let Some(existing) = turns[last]["decisions"].as_array_mut() { + existing.extend(orphan_decisions); + } + } + } + + // Every session produces at least one turn, so an empty tree plus zero + // spans still yields a schema-valid document rather than an empty array. + if turns.is_empty() { + turns.push(json!({ + "turn_id": 1_i64, + "span_id": format!("{session_id}:turn-1"), + "timestamp": timestamp_start, + "error": status == "Failed", + "messages": Vec::::new(), + "decisions": Vec::::new(), + })); + } + + // Announced in a kernel-modeled field so a consumer working from a + // re-serialized row knows the stored row holds signals its structs cannot + // represent. Only when a signal was actually written: a turn whose signals + // were all dropped carries the record of the drop, not a signal to read. + // A signal the writer or this budget refused is evidence the row was built + // without, which is what degraded means — so it reaches the Session status + // the same way a missing transcript does, rather than only the document. + if lost_token_signals { + tags.push(format!("{DEGRADED_TAG_PREFIX}token_signals_dropped")); + } + + // trajectory_id is duplicated inside `metadata` because Temper's server-side + // POST handler at temper-server/src/observe/evolution/trajectories.rs reads + // it from metadata.trajectory_id (not the OTS top-level field). Emitting in + // both places keeps OTS schema compliance AND lets the Turso row use the same + // id my module stored on the Session entity — which is what makes + // INSERT OR REPLACE-based retry idempotency actually work. + let mut metadata = json!({ + "trajectory_id": trajectory_id, + "task_description": task_description, + "domain": "temperpaw-agent", + "timestamp_start": timestamp_start, + "timestamp_end": timestamp_end, + "agent_id": agent_id, + "framework": "temperpaw", + "harness": HARNESS, + "environment": "production", + "outcome": outcome, + "tags": tags, + }); + if !spec_version.is_empty() { + metadata["spec_version"] = json!(spec_version); + } + + let mut resources: Vec = Vec::new(); + file_resource(&mut resources, "session_tree", field_str(fields, "session_file_id")); + file_resource(&mut resources, "tool_spans", field_str(fields, "tool_spans_file_id")); + file_resource( + &mut resources, + "prepared_context", + field_str(fields, "prepared_context_file_id"), + ); + file_resource( + &mut resources, + "provider_response", + field_str(fields, "provider_response_file_id"), + ); + file_resource( + &mut resources, + "system_prompt", + field_str(fields, "system_prompt_file_id"), + ); + + let mut trajectory = json!({ + "trajectory_id": trajectory_id, + "version": OTS_VERSION, + "metadata": metadata, + "turns": turns, + "_token_usage": { + "input_tokens": fields.get("input_tokens").and_then(json_u64).unwrap_or(0), + "output_tokens": fields.get("output_tokens").and_then(json_u64).unwrap_or(0), + "context_tokens": fields.get("context_tokens").and_then(json_u64).unwrap_or(0), + }, + "_session_turn_count": fields.get("turn_count").and_then(json_u64).unwrap_or(0), + }); + + if span_document.truncated { + // The span document hit its ceiling, so some tool timings are missing. + // A consumer training on this must know the record is partial. + trajectory["_tool_spans_truncated"] = json!(true); + } + + if span_document.unparsed_lines > 0 { + // Tool calls whose only evidence was a span line that did not parse. + trajectory["_tool_spans_unparsed_lines"] = json!(span_document.unparsed_lines); + } + + if !transcript_reasons.is_empty() { + // The turn structure that makes a trajectory readable was not all + // there. Each reason is a different way of being short of the session + // that actually ran. + let mut marker = json!({ + "present": transcript.is_present(), + "reasons": transcript_reasons, + }); + if unparsed_lines > 0 { + marker["unparsed_lines"] = json!(unparsed_lines); + } + trajectory["_transcript"] = marker; + } + if tool_spans_missing { + trajectory["_tool_spans_missing"] = json!(true); + } + if tool_spans_write_failed { + trajectory["_tool_spans_write_failed"] = json!(true); + } + + if !resources.is_empty() { + trajectory["context"] = json!({ "resources": resources }); + } + + let system_prompt = field_str(fields, "system_prompt"); + if !system_prompt.is_empty() { + // OTSSystemMessage only has { content, timestamp } — no `role` field. + trajectory["system_message"] = json!({ + "content": truncate_chars(system_prompt, MAX_SYSTEM_PROMPT_CHARS), + "timestamp": trajectory["metadata"]["timestamp_start"].clone(), + }); + } + + trajectory +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The exact set of emitted fields the pinned `temper-ots` structs do not + /// model, and therefore drop on a deserialize/re-serialize round trip. + /// Pinned by `kernel_round_trip_drops_exactly_the_unmodeled_extensions`, + /// which fails the day the kernel starts modeling one of them. + const KERNEL_UNMODELED_FIELDS: &[&str] = &[ + // Read by the server's POST handler before any struct is involved, and + // repeated at the top level where the kernel does model it. + "metadata.trajectory_id", + ]; + + fn inputs<'a>( + fields: &'a Value, + session_jsonl: &'a str, + tool_spans_jsonl: &'a str, + entity_state: &'a Value, + status: &'a str, + ) -> TrajectoryInputs<'a> { + TrajectoryInputs { + trajectory_id: "trj-ss-1", + session_id: "ss-1", + agent_id: "aj-1", + status, + fields, + session_jsonl, + tool_spans_jsonl, + entity_state, + spec_version: "paw-agent@0.1.0", + // Fixtures that pass a transcript describe a session whose + // transcript was read; the degradation paths set this explicitly. + transcript: if session_jsonl.trim().is_empty() { + TranscriptPresence::NoEntries + } else { + TranscriptPresence::Present + }, + tool_spans_missing: false, + } + } + + fn entity_state_with_events() -> Value { + json!({ + "events": [ + { "action": "Created", "timestamp": "2026-01-01T00:00:00Z" }, + { "action": "Cancel", "timestamp": "2026-01-01T00:00:01Z" }, + ] + }) + } + + /// Two real LLM cycles: user -> assistant(tool_use) -> tool_result -> + /// assistant(final text). + fn two_turn_session_jsonl() -> String { + let lines = [ + json!({"id":"h-ss-1","parentId":null,"type":"header","tokens":0}), + json!({ + "id":"u-ss-1-0","parentId":"h-ss-1","type":"message","role":"user", + "content":"find the bug","tokens":3,"ts_ms":1_767_225_600_000_i64 + }), + json!({ + "id":"a-1","parentId":"u-ss-1-0","type":"message","role":"assistant", + "content":[ + {"type":"thinking","thinking":"I should grep first"}, + {"type":"text","text":"Looking now"}, + {"type":"tool_use","id":"tc-1","name":"temper.bash","input":{"cmd":"rg TODO"}} + ], + "tokens":42,"ts_ms":1_767_225_601_000_i64, + "input_tokens":100,"output_tokens":42 + }), + json!({ + "id":"t-2","parentId":"a-1","type":"message","role":"user", + "content":[ + {"type":"tool_result","tool_use_id":"tc-1","content":"src/lib.rs:12: TODO","is_error":false} + ], + "tokens":9,"ts_ms":1_767_225_602_000_i64 + }), + json!({ + "id":"a-3","parentId":"t-2","type":"message","role":"assistant", + "content":[{"type":"text","text":"Found it on line 12"}], + "tokens":11,"ts_ms":1_767_225_603_000_i64, + "input_tokens":160,"output_tokens":11 + }), + ]; + lines + .iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n") + } + + fn two_turn_fields() -> Value { + json!({ + "user_message": "find the bug", + "model": "claude-sonnet-4-6", + "provider": "anthropic", + "session_mode": "execute", + "has_result": true, + "session_leaf_id": "a-3", + "session_file_id": "session-entries:ss-1", + "tool_spans_file_id": "file-spans-1", + "turn_count": 2, + "input_tokens": 260, + "output_tokens": 53, + "context_tokens": 160, + }) + } + + #[test] + fn derive_outcome_matrix() { + assert_eq!(derive_outcome("Completed", true), "success"); + assert_eq!(derive_outcome("Completed", false), "partial_success"); + assert_eq!(derive_outcome("Failed", true), "failure"); + assert_eq!(derive_outcome("Failed", false), "failure"); + assert_eq!(derive_outcome("Cancelled", true), "partial_success"); + assert_eq!(derive_outcome("Cancelled", false), "partial_success"); + assert_eq!(derive_outcome("Thinking", true), "failure"); // defensive + } + + #[test] + fn classify_error_categories() { + assert_eq!(classify_error(""), "unknown_error"); + assert_eq!(classify_error("request timed out after 30s"), "tool_timeout"); + assert_eq!(classify_error("Cedar denied action"), "cedar_denied"); + assert_eq!(classify_error("permission denied by policy"), "cedar_denied"); + assert_eq!(classify_error("subprocess failed"), "tool_error"); + } + + #[test] + fn truncate_chars_respects_utf8() { + let s = "héllo wörld"; + assert_eq!(truncate_chars(s, 5), "héllo"); + assert_eq!(truncate_chars(s, 100), s); + } + + #[test] + fn rfc3339_from_millis_formats_utc() { + assert_eq!(rfc3339_from_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!(rfc3339_from_millis(-5), "1970-01-01T00:00:00.000Z"); + assert_eq!( + rfc3339_from_millis(1_767_225_600_000), + "2026-01-01T00:00:00.000Z" + ); + // Leap day, with sub-second precision retained. + assert_eq!( + rfc3339_from_millis(1_709_209_845_123), + "2024-02-29T12:30:45.123Z" + ); + } + + #[test] + fn span_to_decision_success_shape() { + let span = json!({ + "tool_name": "temper.read", + "tool_call_id": "tc-123", + "arguments": "{\"path\":\"/tmp/foo\"}", + "result": "ok", + "duration_ms": 42, + "is_error": false, + }); + let dec = span_to_decision(&span, &mut InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS)); + assert_eq!(dec["decision_id"], "tc-123"); + assert_eq!(dec["decision_type"], "tool_selection"); + assert_eq!(dec["cause_id"], "tc-123"); + assert_eq!(dec["choice"]["action"], "temper.read"); + assert_eq!(dec["choice"]["arguments"]["path"], "/tmp/foo"); + assert_eq!(dec["consequence"]["success"], true); + assert_eq!(dec["consequence"]["result_summary"], "ok"); + assert!(dec["consequence"].get("error_type").is_none()); + assert_eq!(dec["_duration_ms"], 42); + } + + #[test] + fn span_to_decision_error_classifies() { + let span = json!({ + "tool_name": "temper.bash", + "tool_call_id": "tc-err", + "arguments": "{}", + "result": "Cedar denied bash execution", + "duration_ms": 5, + "is_error": true, + }); + let dec = span_to_decision(&span, &mut InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS)); + assert_eq!(dec["consequence"]["success"], false); + assert_eq!(dec["consequence"]["error_type"], "cedar_denied"); + } + + #[test] + fn span_to_decision_keeps_non_json_arguments_as_string() { + let span = json!({ + "tool_name": "raw", + "tool_call_id": "tc-raw", + "arguments": "not-valid-json", + "result": "", + "duration_ms": 0, + "is_error": false, + }); + let dec = span_to_decision(&span, &mut InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS)); + assert_eq!(dec["choice"]["arguments"], "not-valid-json"); + } + + #[test] + fn span_to_decision_truncates_long_result() { + let long_result = "x".repeat(1000); + let span = json!({ + "tool_name": "temper.bash", + "tool_call_id": "tc-long", + "arguments": "{}", + "result": long_result, + "duration_ms": 1, + "is_error": false, + }); + let dec = span_to_decision(&span, &mut InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS)); + let summary = dec["consequence"]["result_summary"].as_str().unwrap(); + assert_eq!(summary.chars().count(), MAX_RESULT_SUMMARY_CHARS); + } + + #[test] + fn parse_tool_spans_skips_invalid_lines() { + let jsonl = "{\"tool_call_id\":\"a\",\"tool_name\":\"x\",\"result\":\"\",\"duration_ms\":0,\"is_error\":false}\nINVALID\n{\"tool_call_id\":\"b\",\"tool_name\":\"y\",\"result\":\"\",\"duration_ms\":0,\"is_error\":false}\n"; + let mut budget = InlineBudget::new(MAX_TRAJECTORY_INLINE_CHARS); + let decisions: Vec = parse_tool_span_document(jsonl) + .spans + .iter() + .map(|span| span_to_decision(span, &mut budget)) + .collect(); + assert_eq!(decisions.len(), 2); + assert_eq!(decisions[0]["decision_id"], "a"); + assert_eq!(decisions[1]["decision_id"], "b"); + } + + #[test] + fn resolve_chain_prefers_recorded_leaf() { + let entries = parse_session_entries(&two_turn_session_jsonl()); + let resolved = resolve_chain(&entries, "a-3"); + let ids: Vec<&str> = resolved + .chain + .iter() + .map(|i| entries[*i].id.as_str()) + .collect(); + assert_eq!(ids, vec!["h-ss-1", "u-ss-1-0", "a-1", "t-2", "a-3"]); + assert!(resolved.from_recorded_leaf); + } + + #[test] + fn resolve_chain_falls_back_when_leaf_is_missing() { + let entries = parse_session_entries(&two_turn_session_jsonl()); + let resolved = resolve_chain(&entries, "a-999-never-written"); + let ids: Vec<&str> = resolved + .chain + .iter() + .map(|i| entries[*i].id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["h-ss-1", "u-ss-1-0", "a-1", "t-2", "a-3"], + "a leaf ahead of durable rows must not empty the trajectory" + ); + assert!( + !resolved.from_recorded_leaf, + "the fallback chain is missing the newest turns, and the caller has to know" + ); + } + + /// The names the carrier and the drop markers iterate must stay the same + /// set the turn writer emits, or a signal added later travels silently. + #[test] + fn token_signal_fields_cover_every_signal() { + let mut expected: Vec<&str> = vec!["prompt_token_ids"]; + expected.extend(COMPLETION_TOKEN_SIGNALS.iter().map(|(field, _)| *field)); + let mut expected: Vec<&str> = expected; + expected.sort_unstable(); + let mut declared: Vec<&str> = TOKEN_SIGNAL_FIELDS.to_vec(); + declared.sort_unstable(); + assert_eq!(declared, expected); + } + + /// A cycle is not a chain that stops early: everything above the loop is + /// unreachable, so accepting the fragment would report a truncated history + /// as the recorded leaf's own. + #[test] + fn resolve_chain_reports_a_cyclic_ancestry_as_unresolved() { + let jsonl = [ + json!({"id":"u-0","parentId":null,"type":"message","role":"user","content":"go"}), + json!({"id":"a-0","parentId":"u-0","type":"message","role":"assistant","content":"ok"}), + json!({"id":"a","parentId":"b","type":"message","role":"assistant","content":"x"}), + json!({"id":"b","parentId":"a","type":"message","role":"user","content":"y"}), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let entries = parse_session_entries(&jsonl); + let resolved = resolve_chain(&entries, "a"); + assert!( + !resolved.from_recorded_leaf, + "a leaf whose ancestry loops has not resolved" + ); + + let fields = json!({ "session_leaf_id": "a", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + assert!( + degradations(&t).contains(&"transcript_leaf_unresolved".to_string()), + "tags: {:?}", + t["metadata"]["tags"] + ); + } + + #[test] + fn resolve_chain_survives_parent_cycle() { + let jsonl = [ + json!({"id":"a","parentId":"b","type":"message","role":"assistant","content":"x"}), + json!({"id":"b","parentId":"a","type":"message","role":"user","content":"y"}), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let entries = parse_session_entries(&jsonl); + let resolved = resolve_chain(&entries, "a"); + assert!(resolved.chain.len() <= entries.len()); + } + + #[test] + fn build_trajectory_reconstructs_real_turns() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"arguments\":\"{}\",\"result\":\"src/lib.rs:12: TODO\",\"duration_ms\":137,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let turns = t["turns"].as_array().unwrap(); + assert_eq!(turns.len(), 2, "two assistant messages means two turns"); + assert_eq!(turns[0]["turn_id"], 1); + assert_eq!(turns[1]["turn_id"], 2); + assert_eq!(turns[0]["span_id"], "ss-1:a-1"); + assert_eq!(turns[1]["span_id"], "ss-1:a-3"); + assert_eq!(turns[0]["timestamp"], "2026-01-01T00:00:01.000Z"); + assert_eq!(turns[1]["timestamp"], "2026-01-01T00:00:03.000Z"); + + // Turn 1 carries the user prompt plus the assistant reply. + let turn1_messages = turns[0]["messages"].as_array().unwrap(); + assert_eq!(turn1_messages.len(), 2); + assert_eq!(turn1_messages[0]["role"], "user"); + assert_eq!(turn1_messages[0]["content"]["text"], "find the bug"); + assert_eq!(turn1_messages[1]["role"], "assistant"); + assert_eq!(turn1_messages[1]["content"]["type"], "tool_call"); + assert_eq!(turn1_messages[1]["content"]["text"], "Looking now"); + assert_eq!(turn1_messages[1]["reasoning"], "I should grep first"); + + // Turn 2's prompt side is the tool result observation. + let turn2_messages = turns[1]["messages"].as_array().unwrap(); + assert_eq!(turn2_messages.len(), 2); + assert_eq!(turn2_messages[0]["role"], "tool"); + assert_eq!(turn2_messages[0]["content"]["type"], "tool_response"); + assert_eq!( + turn2_messages[0]["content"]["data"]["tool_results"][0]["tool_call_id"], + "tc-1" + ); + } + + #[test] + fn build_trajectory_populates_decisions_with_cause_id() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"arguments\":\"{}\",\"result\":\"ignored\",\"duration_ms\":137,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let decisions = t["turns"][0]["decisions"].as_array().unwrap(); + assert_eq!(decisions.len(), 1); + assert_eq!(decisions[0]["decision_id"], "tc-1"); + assert_eq!(decisions[0]["cause_id"], "tc-1"); + assert_eq!(decisions[0]["decision_type"], "tool_selection"); + assert_eq!(decisions[0]["choice"]["action"], "temper.bash"); + assert_eq!(decisions[0]["choice"]["arguments"]["cmd"], "rg TODO"); + assert_eq!(decisions[0]["consequence"]["success"], true); + assert_eq!( + decisions[0]["consequence"]["result_summary"], "src/lib.rs:12: TODO", + "the tool_result observation wins over the span result" + ); + assert_eq!(decisions[0]["_duration_ms"], 137); + assert_eq!(t["turns"][0]["duration_ms"], 137.0); + assert_eq!(t["turns"][0]["error"], false); + assert!( + t["turns"][1]["decisions"].as_array().unwrap().is_empty(), + "the final text-only turn makes no tool decisions" + ); + } + + #[test] + fn build_trajectory_marks_turn_error_on_failed_tool_call() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"tool_use","id":"tc-9","name":"temper.bash","input":{}}] + }), + json!({ + "id":"t-2","parentId":"a-1","type":"message","role":"user", + "content":[{"type":"tool_result","tool_use_id":"tc-9","content":"Cedar denied bash","is_error":true}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "t-2", "has_result": false }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Failed")); + + assert_eq!(t["turns"][0]["error"], true); + assert_eq!(t["turns"][0]["decisions"][0]["consequence"]["success"], false); + assert_eq!( + t["turns"][0]["decisions"][0]["consequence"]["error_type"], + "cedar_denied" + ); + } + + #[test] + fn build_trajectory_references_externalized_content_instead_of_inlining() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content_file_id":"file-abc","content_file_version_id":"ver-1","tokens":9000 + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let assistant = &t["turns"][0]["messages"][1]; + assert_eq!(assistant["content"]["data"]["content_file_id"], "file-abc"); + assert_eq!( + assistant["content"]["data"]["content_file_version_id"], + "ver-1" + ); + assert_eq!(assistant["content"]["data"]["externalized"], true); + assert!( + assistant["content"].get("text").is_none(), + "externalized bodies must never be inlined" + ); + } + + #[test] + fn build_trajectory_bounds_inline_text() { + let huge = "x".repeat(MAX_MESSAGE_INLINE_CHARS * 3); + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":huge}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let user = &t["turns"][0]["messages"][0]; + let text = user["content"]["text"].as_str().unwrap(); + assert_eq!(text.chars().count(), MAX_MESSAGE_INLINE_CHARS); + assert_eq!( + user["content"]["data"]["truncated_chars"], + json!(MAX_MESSAGE_INLINE_CHARS * 2) + ); + } + + #[test] + fn build_trajectory_bounds_total_inline_text_across_messages() { + let chunk = "y".repeat(MAX_MESSAGE_INLINE_CHARS); + let mut lines: Vec = vec![json!({ + "id":"u-0","parentId":null,"type":"message","role":"user","content":"start" + })]; + let mut parent = "u-0".to_string(); + // 40 assistant/user pairs of full-size bodies far exceed the global budget. + for index in 1..40 { + let assistant = format!("a-{index}"); + lines.push(json!({ + "id": assistant, "parentId": parent, "type": "message", "role": "assistant", + "content": [{"type":"text","text": chunk}] + })); + let user = format!("u-{index}"); + lines.push(json!({ + "id": user, "parentId": assistant, "type": "message", "role": "user", + "content": chunk + })); + parent = user; + } + let jsonl = lines + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": parent, "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let inline_chars: usize = t["turns"] + .as_array() + .unwrap() + .iter() + .flat_map(|turn| turn["messages"].as_array().unwrap()) + .filter_map(|message| message["content"].get("text").and_then(Value::as_str)) + .map(|text| text.chars().count()) + .sum(); + assert!( + inline_chars <= MAX_TRAJECTORY_INLINE_CHARS, + "inline text budget exceeded: {inline_chars}" + ); + } + + #[test] + fn build_trajectory_carries_token_signals_when_recorded() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "input_tokens": 12, "output_tokens": 3, + "prompt_token_ids":[1,2,3], + "completion_token_ids":[4,5], + "response_mask":[1,1], + "logprobs":[-0.25,-1.5] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let turn = &t["turns"][0]; + assert_eq!(turn["prompt_token_ids"], json!([1, 2, 3])); + assert_eq!(turn["completion_token_ids"], json!([4, 5])); + assert_eq!(turn["response_mask"], json!([1, 1])); + assert_eq!(turn["logprobs"], json!([-0.25, -1.5])); + assert_eq!(turn["_prompt_tokens"], 12); + assert_eq!(turn["_completion_tokens"], 3); + } + + #[test] + fn build_trajectory_omits_malformed_token_signals() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "prompt_token_ids":["not","numbers"], + "response_mask":[7000], + "logprobs":"nope" + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let turn = &t["turns"][0]; + assert!(turn.get("prompt_token_ids").is_none()); + assert!(turn.get("response_mask").is_none()); + assert!(turn.get("logprobs").is_none()); + } + + #[test] + fn build_trajectory_sets_contract_metadata() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!(t["trajectory_id"], "trj-ss-1"); + assert_eq!(t["version"], "0.1.0"); + assert_eq!(t["metadata"]["trajectory_id"], "trj-ss-1"); + assert_eq!(t["metadata"]["harness"], "temperpaw"); + assert_eq!(t["metadata"]["spec_version"], "paw-agent@0.1.0"); + assert_eq!(t["metadata"]["framework"], "temperpaw"); + assert_eq!(t["metadata"]["agent_id"], "aj-1"); + assert_eq!(t["metadata"]["outcome"], "success"); + assert_eq!(t["metadata"]["domain"], "temperpaw-agent"); + assert_eq!(t["metadata"]["task_description"], "find the bug"); + assert_eq!( + t["metadata"]["timestamp_start"], "2026-01-01T00:00:00.000Z", + "the first entry's own timestamp beats the event hot tail" + ); + assert_eq!(t["metadata"]["timestamp_end"], "2026-01-01T00:00:01Z"); + let tags = t["metadata"]["tags"].as_array().unwrap(); + assert!(tags.iter().any(|v| v == "claude-sonnet-4-6")); + assert!(tags.iter().any(|v| v == "anthropic")); + assert!(tags.iter().any(|v| v == "execute")); + assert_eq!(t["_session_turn_count"], 2); + assert_eq!(t["_token_usage"]["input_tokens"], 260); + assert_eq!(t["_token_usage"]["output_tokens"], 53); + assert_eq!(t["_token_usage"]["context_tokens"], 160); + } + + #[test] + fn build_trajectory_references_session_artifacts_as_resources() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let resources = t["context"]["resources"].as_array().unwrap(); + let kinds: Vec<&str> = resources + .iter() + .map(|r| r["type"].as_str().unwrap()) + .collect(); + assert!(kinds.contains(&"session_tree")); + assert!(kinds.contains(&"tool_spans")); + assert_eq!( + resources + .iter() + .find(|r| r["type"] == "tool_spans") + .unwrap()["uri"], + "temperfs://Files('file-spans-1')" + ); + assert_eq!( + resources + .iter() + .find(|r| r["type"] == "session_tree") + .unwrap()["uri"], + "temper://SessionEntries?SessionId=ss-1", + "an entity-backed transcript must not be advertised as a TemperFS file" + ); + } + + #[test] + fn build_trajectory_reports_a_sealed_span_document_without_faking_a_decision() { + let spans = concat!( + "{\"tool_call_id\":\"tc-a\",\"tool_name\":\"read\",\"result\":\"ok\",\"duration_ms\":1,\"is_error\":false}\n", + "{\"tool_call_id\":\"\",\"tool_name\":\"_tool_spans_truncated\",\"result\":\"tool span file size ceiling reached\",\"duration_ms\":0,\"is_error\":false}\n", + ); + let fields = json!({ "user_message": "x", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", spans, &state, "Completed")); + + assert_eq!(t["_tool_spans_truncated"], true); + let decisions = t["turns"][0]["decisions"].as_array().unwrap(); + assert_eq!(decisions.len(), 1, "the marker must not become a decision"); + assert_eq!(decisions[0]["decision_id"], "tc-a"); + } + + #[test] + fn build_trajectory_keeps_unclaimed_spans_as_decisions() { + // Assistant body externalized: the tree cannot name the tool call, so + // the span is the only evidence and must still become a decision. + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content_file_id":"file-xyz","tokens":900 + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let spans = "{\"tool_call_id\":\"tc-orphan\",\"tool_name\":\"temper.read\",\"arguments\":\"{}\",\"result\":\"ok\",\"duration_ms\":4,\"is_error\":false}\n"; + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let decisions = t["turns"][0]["decisions"].as_array().unwrap(); + assert_eq!(decisions.len(), 1); + assert_eq!(decisions[0]["decision_id"], "tc-orphan"); + assert_eq!(decisions[0]["cause_id"], "tc-orphan"); + } + + #[test] + fn build_trajectory_without_session_tree_falls_back_to_spans() { + let fields = json!({ "user_message": "x", "has_result": false }); + let spans = "{\"tool_call_id\":\"tc-a\",\"tool_name\":\"read\",\"arguments\":\"{}\",\"result\":\"ok\",\"duration_ms\":2,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", spans, &state, "Cancelled")); + + assert_eq!(t["metadata"]["outcome"], "partial_success"); + let turns = t["turns"].as_array().unwrap(); + assert_eq!(turns.len(), 1); + assert_eq!(turns[0]["decisions"].as_array().unwrap().len(), 1); + assert_eq!(turns[0]["decisions"][0]["decision_id"], "tc-a"); + } + + #[test] + fn build_trajectory_empty_session_still_emits_one_turn() { + let fields = json!({ "user_message": "", "has_result": false }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", "", &state, "Failed")); + + assert_eq!(t["metadata"]["outcome"], "failure"); + let turns = t["turns"].as_array().unwrap(); + assert_eq!(turns.len(), 1); + assert_eq!(turns[0]["turn_id"], 1); + assert_eq!(turns[0]["error"], true); + assert!(turns[0]["decisions"].as_array().unwrap().is_empty()); + assert!( + turns[0]["timestamp"].as_str().is_some_and(|s| !s.is_empty()), + "turn timestamp must be non-empty" + ); + } + + #[test] + fn build_trajectory_includes_system_message_when_present() { + let fields = json!({ "user_message": "", "system_prompt": "you are a helpful agent" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", "", &state, "Completed")); + // OTSSystemMessage has { content, timestamp } only — no `role` field. + assert_eq!(t["system_message"]["content"], "you are a helpful agent"); + assert!( + t["system_message"]["timestamp"] + .as_str() + .is_some_and(|s| !s.is_empty()), + "system_message.timestamp must be non-empty" + ); + assert!( + t["system_message"].get("role").is_none(), + "role must be omitted" + ); + } + + #[test] + fn build_trajectory_skips_system_message_when_empty() { + let fields = json!({ "user_message": "", "system_prompt": "" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", "", &state, "Completed")); + assert!(t.get("system_message").is_none()); + } + + #[test] + fn build_trajectory_missing_events_emits_epoch_fallback() { + let fields = json!({ "user_message": "x" }); + let t = build_trajectory(&inputs(&fields, "", "", &json!({}), "Completed")); + assert_eq!(t["metadata"]["timestamp_start"], "1970-01-01T00:00:00Z"); + assert_eq!(t["metadata"]["timestamp_end"], "1970-01-01T00:00:00Z"); + } + + #[test] + fn build_trajectory_uses_event_log_when_entries_lack_timestamps() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let state = json!({ + "events": [ + { "action": "Created", "timestamp": "2026-03-01T00:00:00Z" }, + { "action": "RecordResult", "timestamp": "2026-03-01T00:00:09Z" }, + ] + }); + let fields = json!({ "session_leaf_id": "a-1" }); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + assert_eq!(t["turns"][0]["timestamp"], "2026-03-01T00:00:09Z"); + } + + /// The wire contract with the kernel. If a field name or type drifts, the + /// POST at `/api/ots/trajectories` stores a row that no consumer can read; + /// this catches it at build time instead. + #[test] + fn build_trajectory_deserializes_as_a_kernel_ots_trajectory() { + use temper_ots::models::{ContentType, DecisionType, MessageRole, OTSTrajectory, OutcomeType}; + + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"arguments\":\"{}\",\"result\":\"src/lib.rs:12: TODO\",\"duration_ms\":137,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let document = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let trajectory: OTSTrajectory = serde_json::from_value(document) + .expect("emitted document must deserialize as the kernel OTSTrajectory"); + + assert_eq!(trajectory.trajectory_id, "trj-ss-1"); + assert_eq!(trajectory.version, OTS_VERSION); + assert_eq!(trajectory.metadata.outcome, OutcomeType::Success); + assert_eq!(trajectory.metadata.agent_id, "aj-1"); + assert_eq!(trajectory.metadata.framework.as_deref(), Some("temperpaw")); + assert_eq!(trajectory.turns.len(), 2); + + let first = &trajectory.turns[0]; + assert_eq!(first.turn_id, 1); + assert_eq!(first.messages.len(), 2); + assert_eq!(first.messages[0].role, MessageRole::User); + assert_eq!(first.messages[1].role, MessageRole::Assistant); + assert_eq!(first.messages[1].content.content_type, ContentType::ToolCall); + assert_eq!( + first.messages[1].reasoning.as_deref(), + Some("I should grep first") + ); + assert_eq!(first.decisions.len(), 1); + assert_eq!(first.decisions[0].decision_id, "tc-1"); + assert_eq!(first.decisions[0].decision_type, DecisionType::ToolSelection); + assert_eq!(first.decisions[0].choice.action, "temper.bash"); + assert!(first.decisions[0].consequence.success); + assert_eq!(first.duration_ms, Some(137.0)); + + let second = &trajectory.turns[1]; + assert_eq!(second.turn_id, 2); + assert_eq!(second.messages[0].role, MessageRole::Tool); + assert_eq!( + second.messages[0].content.content_type, + ContentType::ToolResponse + ); + + // Session artifacts survive as context resources rather than payloads. + assert!( + trajectory + .context + .resources + .iter() + .any(|resource| resource.resource_type == "tool_spans"), + "tool span file must be referenced in the trajectory context" + ); + } + + /// Terminal states other than success have to survive the same round trip — + /// a failed run is training signal, not a row the consumer can skip. #[test] - fn span_to_decision_keeps_non_json_arguments_as_string() { - let span = json!({ - "tool_name": "raw", - "tool_call_id": "tc-raw", - "arguments": "not-valid-json", - "result": "", - "duration_ms": 0, - "is_error": false, - }); - let dec = span_to_decision(&span); - assert_eq!(dec["choice"]["arguments"], "not-valid-json"); + fn failed_and_cancelled_trajectories_also_deserialize() { + use temper_ots::models::{OTSTrajectory, OutcomeType}; + + let fields = json!({ "user_message": "x", "has_result": false }); + let state = entity_state_with_events(); + for (status, expected) in [ + ("Failed", OutcomeType::Failure), + ("Cancelled", OutcomeType::PartialSuccess), + ] { + let document = build_trajectory(&inputs(&fields, "", "", &state, status)); + let trajectory: OTSTrajectory = serde_json::from_value(document) + .unwrap_or_else(|err| panic!("{status} document must deserialize: {err}")); + assert_eq!(trajectory.metadata.outcome, expected); + assert_eq!(trajectory.turns.len(), 1); + } } + /// Tool arguments are the largest field a decision can carry — a file-write + /// call holds the whole file — so many of them must not inflate the document + /// past the global ceiling. #[test] - fn span_to_decision_truncates_long_result() { - let long_result = "x".repeat(1000); + fn build_trajectory_counts_tool_arguments_against_the_global_budget() { + let payload = "q".repeat(MAX_ARGUMENTS_CHARS); + let mut lines: Vec = vec![json!({ + "id":"u-0","parentId":null,"type":"message","role":"user","content":"go" + })]; + let mut parent = "u-0".to_string(); + for turn in 1..40 { + let assistant = format!("a-{turn}"); + lines.push(json!({ + "id": assistant, "parentId": parent, "type": "message", "role": "assistant", + "content": [{ + "type": "tool_use", + "id": format!("tc-{turn}"), + "name": "temper.write", + "input": {"body": payload} + }] + })); + let tool = format!("t-{turn}"); + lines.push(json!({ + "id": tool, "parentId": assistant, "type": "message", "role": "user", + "content": [{"type":"tool_result","tool_use_id":format!("tc-{turn}"),"content":"ok"}] + })); + parent = tool; + } + let jsonl = lines + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": parent, "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let argument_chars: usize = t["turns"] + .as_array() + .unwrap() + .iter() + .flat_map(|turn| turn["decisions"].as_array().unwrap()) + .filter_map(|decision| decision["choice"].get("arguments")) + .map(|arguments| { + serde_json::to_string(arguments) + .unwrap_or_default() + .chars() + .count() + }) + .sum(); + assert!( + argument_chars <= MAX_TRAJECTORY_INLINE_CHARS * 2, + "tool arguments escaped the inline budget: {argument_chars} chars" + ); + assert_eq!( + t["turns"][30]["decisions"][0]["choice"]["arguments"]["_truncated"], + json!(true), + "late decisions must degrade to a preview once the budget is spent" + ); + assert!( + t["turns"][0]["messages"][1]["content"]["data"]["tool_calls"][0] + .get("arguments") + .is_none(), + "the message must not duplicate the decision's arguments" + ); + } + + /// Providers that omit tool-call ids get synthetic ones that restart at + /// every response, so two turns can both call `tool_1`. Indexing spans and + /// observations across the whole document made the second call overwrite the + /// first: both decisions then carried the second call's result and duration. + #[test] + fn build_trajectory_keeps_repeated_tool_call_ids_apart_per_turn() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"tool_use","id":"tool_1","name":"temper.read","input":{"path":"/first"}}] + }), + json!({ + "id":"t-1","parentId":"a-1","type":"message","role":"user", + "content":[{"type":"tool_result","tool_use_id":"tool_1","content":"first result","is_error":false}] + }), + json!({ + "id":"a-2","parentId":"t-1","type":"message","role":"assistant", + "content":[{"type":"tool_use","id":"tool_1","name":"temper.bash","input":{"cmd":"second"}}] + }), + json!({ + "id":"t-2","parentId":"a-2","type":"message","role":"user", + "content":[{"type":"tool_result","tool_use_id":"tool_1","content":"second failed","is_error":true}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let spans = concat!( + "{\"tool_call_id\":\"tool_1\",\"tool_name\":\"temper.read\",\"result\":\"first result\",\"duration_ms\":11,\"is_error\":false}\n", + "{\"tool_call_id\":\"tool_1\",\"tool_name\":\"temper.bash\",\"result\":\"second failed\",\"duration_ms\":22,\"is_error\":true}\n", + ); + let fields = json!({ "session_leaf_id": "t-2", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let first = &t["turns"][0]["decisions"][0]; + let second = &t["turns"][1]["decisions"][0]; + assert_eq!(first["choice"]["action"], "temper.read"); + assert_eq!(first["consequence"]["result_summary"], "first result"); + assert_eq!(first["consequence"]["success"], true); + assert_eq!(first["_duration_ms"], 11); + assert_eq!(second["choice"]["action"], "temper.bash"); + assert_eq!(second["consequence"]["result_summary"], "second failed"); + assert_eq!(second["consequence"]["success"], false); + assert_eq!(second["_duration_ms"], 22); + assert_eq!(t["turns"][0]["error"], false); + assert_eq!(t["turns"][1]["error"], true); + assert_eq!( + t["turns"][0]["duration_ms"], 11.0, + "a repeated id must not double-count one span onto two turns" + ); + } + + /// Two spans sharing an id and no transcript to claim them are two calls, + /// not one — collapsing them would erase a whole tool execution. + #[test] + fn build_trajectory_keeps_repeated_ids_among_unclaimed_spans() { + let spans = concat!( + "{\"tool_call_id\":\"tool_1\",\"tool_name\":\"read\",\"result\":\"a\",\"duration_ms\":1,\"is_error\":false}\n", + "{\"tool_call_id\":\"tool_1\",\"tool_name\":\"bash\",\"result\":\"b\",\"duration_ms\":2,\"is_error\":false}\n", + ); + let fields = json!({ "user_message": "x", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", spans, &state, "Completed")); + + let decisions = t["turns"][0]["decisions"].as_array().unwrap(); + assert_eq!(decisions.len(), 2); + assert_eq!(decisions[0]["choice"]["action"], "read"); + assert_eq!(decisions[1]["choice"]["action"], "bash"); + } + + /// A tool that reads or greps the emitter's own source returns the marker + /// literal in its result. That must not make a complete run look partial. + #[test] + fn build_trajectory_does_not_read_a_quoted_truncation_marker_as_truncation() { let span = json!({ - "tool_name": "temper.bash", - "tool_call_id": "tc-long", - "arguments": "{}", - "result": long_result, + "tool_call_id": "tc-a", + "tool_name": "temper.read", + "result": format!("pub const MARKER: &str = \"{TOOL_SPANS_TRUNCATED_MARKER}\";"), "duration_ms": 1, "is_error": false, }); - let dec = span_to_decision(&span); - let summary = dec["consequence"]["result_summary"].as_str().unwrap(); - assert_eq!(summary.len(), 500); + let spans = format!("{span}\n"); + let fields = json!({ "user_message": "x", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, "", &spans, &state, "Completed")); + + assert!( + t.get("_tool_spans_truncated").is_none(), + "truncation is a reserved tool_name, not a substring of any result" + ); + assert_eq!(t["turns"][0]["decisions"].as_array().unwrap().len(), 1); + assert!(!parse_tool_span_document(&spans).truncated); } + /// Completion-side signals are positionally aligned with one another. + /// Emitting them at different lengths would silently pair probabilities and + /// mask bits with the wrong tokens. #[test] - fn parse_tool_spans_skips_invalid_lines() { - let jsonl = "{\"tool_call_id\":\"a\",\"tool_name\":\"x\",\"result\":\"\",\"duration_ms\":0,\"is_error\":false}\nINVALID\n{\"tool_call_id\":\"b\",\"tool_name\":\"y\",\"result\":\"\",\"duration_ms\":0,\"is_error\":false}\n"; - let decisions = parse_tool_spans_to_decisions(jsonl); - assert_eq!(decisions.len(), 2); - assert_eq!(decisions[0]["decision_id"], "a"); - assert_eq!(decisions[1]["decision_id"], "b"); + fn build_trajectory_drops_misaligned_completion_token_signals() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "prompt_token_ids":[1,2,3], + "completion_token_ids":[4,5,6], + "response_mask":[1,1,1], + "logprobs":[-0.1,-0.2] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let turn = &t["turns"][0]; + assert_eq!( + turn["prompt_token_ids"], + json!([1, 2, 3]), + "the prompt side does not index into the completion and still travels" + ); + for field in ["completion_token_ids", "response_mask", "logprobs"] { + assert!( + turn.get(field).is_none(), + "{field} must not ship in a misaligned set" + ); + } + assert_eq!(turn["_token_signals_misaligned"]["logprobs"], json!(2)); + assert_eq!(turn["_token_signals_misaligned"]["response_mask"], json!(3)); } + /// A completion-side signal with no `completion_token_ids` to index into is + /// dropped, not carried. The kernel rejects that turn outright, and a + /// rejected document is lost rather than degraded: the POST answers 400 and + /// every retry rebuilds the identical document. + /// + /// This is reachable, not hypothetical — the SessionEntry writer bounds each + /// signal on its own, and the ids are several times the size of the mask, so + /// a long completion loses the ids and keeps the mask. #[test] - fn build_trajectory_minimum_shape() { - let fields = json!({ - "user_message": "please find the bug", - "model": "claude-sonnet-4-6", - "provider": "anthropic", - "has_result": true, - "session_mode": "execute", - }); - let jsonl = "{\"tool_call_id\":\"a\",\"tool_name\":\"read\",\"arguments\":\"{}\",\"result\":\"ok\",\"duration_ms\":10,\"is_error\":false}\n"; - let t = build_trajectory( - "trj-42", - "ss-77", - "aj-99", - "Completed", - &fields, - jsonl, - &entity_state_with_events(), - ); - assert_eq!(t["trajectory_id"], "trj-42"); - assert_eq!(t["metadata"]["trajectory_id"], "trj-42"); - assert_eq!(t["version"], "0.1.0"); - assert_eq!(t["metadata"]["outcome"], "success"); - assert_eq!(t["metadata"]["agent_id"], "aj-99"); - assert_eq!(t["metadata"]["domain"], "temperpaw-agent"); - assert_eq!(t["metadata"]["framework"], "temperpaw"); - assert_eq!(t["metadata"]["task_description"], "please find the bug"); - let tags = t["metadata"]["tags"].as_array().unwrap(); - assert!(tags.iter().any(|v| v == "claude-sonnet-4-6")); - assert!(tags.iter().any(|v| v == "anthropic")); - assert!(tags.iter().any(|v| v == "execute")); - let turns = t["turns"].as_array().unwrap(); - assert_eq!(turns.len(), 1); - assert_eq!(turns[0]["turn_id"], 1, "turn_id must be integer per OTSTurn schema"); - assert_eq!(turns[0]["span_id"], "ss-77"); + fn build_trajectory_drops_completion_signals_with_no_anchor() { + for (name, signal) in [ + ("logprobs", json!([-0.1, -0.2])), + ("response_mask", json!([1, 1])), + ] { + let mut assistant = json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + }); + assistant[name] = signal; + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + assistant, + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!( + t["turns"][0].get(name).is_none(), + "{name} has nothing to align against and must not be written" + ); + assert!( + t["turns"][0]["_token_signals_dropped"][name].is_number(), + "the drop must be recorded: {}", + t["turns"][0] + ); + assert!( + degradations(&t).contains(&"token_signals_dropped".to_string()), + "and must reach the Session status" + ); + } + } + + /// The anchor itself travels alone: it is what the others index into, so + /// nothing is missing. + #[test] + fn build_trajectory_keeps_a_lone_completion_token_ids_signal() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "completion_token_ids":[7,8] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!(t["turns"][0]["completion_token_ids"], json!([7, 8])); + assert!(t["turns"][0].get("_token_signals_dropped").is_none()); + } + + /// `response_mask` is a loss switch: the kernel rejects a turn over a single + /// entry above 1, so a mask carrying one is not written at all. + #[test] + fn build_trajectory_refuses_a_response_mask_that_is_not_binary() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "completion_token_ids":[7,8], + "response_mask":[1,2] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!(t["turns"][0].get("response_mask").is_none()); + assert_eq!( + t["turns"][0]["completion_token_ids"], + json!([7, 8]), + "the anchor is still valid on its own" + ); + } + + /// The whole point of the three rules above: whatever the emitter builds, + /// the pinned kernel accepts it. A rejected document is not a partial write + /// — it is a row that never lands and that no retry can change. + #[test] + fn every_asymmetric_signal_shape_still_deserializes_as_a_kernel_turn() { + use temper_ots::models::OTSTrajectory; + + let shapes = [ + json!({"logprobs": [-0.1, -0.2]}), + json!({"response_mask": [1, 1]}), + json!({"response_mask": [1, 2], "completion_token_ids": [7, 8]}), + json!({"logprobs": [-0.1], "completion_token_ids": [7, 8]}), + json!({"response_mask": [1, 1], "logprobs": [-0.1, -0.2]}), + json!({"completion_token_ids": [7, 8], "response_mask": [1, 1], "logprobs": [-0.1, -0.2]}), + json!({"prompt_token_ids": [1, 2, 3], "response_mask": [1, 1]}), + ]; + for shape in shapes { + let mut assistant = json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + }); + for (key, value) in shape.as_object().unwrap() { + assistant[key] = value.clone(); + } + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + assistant, + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let document = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + serde_json::from_value::(document.clone()).unwrap_or_else(|err| { + panic!( + "the kernel refuses a document this emitter built from {shape}: {err}\n\ + the POST would answer 400 and every retry would rebuild it\n{document}" + ) + }); + } + } + + /// The decision-to-observation join must not depend on a field the kernel + /// drops. `cause_id` mirrors `decision_id`, which the kernel does model, so + /// a consumer working from re-serialized rows can still join. + #[test] + fn cause_id_mirrors_the_kernel_modeled_decision_id() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"result\":\"ok\",\"duration_ms\":3,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let mut checked = 0; + for turn in t["turns"].as_array().unwrap() { + for decision in turn["decisions"].as_array().unwrap() { + assert_eq!( + decision["cause_id"], decision["decision_id"], + "the join key must stay a kernel-modeled field" + ); + checked += 1; + } + } + assert!(checked > 0, "the fixture must contain decisions"); + } + + /// Session JSONL whose single assistant turn carries every token signal. + fn token_signal_session_jsonl(tokens: usize) -> String { + [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"done"}], + "prompt_token_ids": vec![7_u64; tokens], + "completion_token_ids": vec![3_u64; tokens], + "response_mask": vec![1_u64; tokens], + "logprobs": vec![-0.5_f64; tokens] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n") + } + + /// Token signals are the one payload the character budgets do not bound, so + /// a long session could otherwise carry many megabytes of them. + #[test] + fn build_trajectory_bounds_token_signals_across_the_document() { + // Two turns of roughly a megabyte of signals each: the first fits, the + // second cannot, and the drop has to be visible on both sides. + let per_turn = MAX_TOKEN_SIGNAL_BYTES / 8; + let mut lines: Vec = vec![ + json!({"id":"u-0","parentId":null,"type":"message","role":"user","content":"go"}), + ]; + let mut parent = "u-0".to_string(); + for turn in 1..=2 { + let assistant = format!("a-{turn}"); + lines.push(json!({ + "id": assistant, "parentId": parent, "type": "message", "role": "assistant", + "content": [{"type":"text","text":"ok"}], + "completion_token_ids": vec![1234_u64; per_turn], + "response_mask": vec![1_u64; per_turn], + })); + parent = assistant; + } + let jsonl = lines + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": parent, "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let signal_bytes: usize = t["turns"] + .as_array() + .unwrap() + .iter() + .flat_map(|turn| { + ["prompt_token_ids", "completion_token_ids", "response_mask", "logprobs"] + .into_iter() + .filter_map(|field| turn.get(field)) + .map(|value| value.to_string().len()) + }) + .sum(); assert!( - turns[0]["timestamp"].as_str().is_some_and(|s| !s.is_empty()), - "turn timestamp must be non-empty" + signal_bytes <= MAX_TOKEN_SIGNAL_BYTES, + "token signals must stay under the trajectory ceiling, got {signal_bytes}" + ); + + let dropped = &t["turns"][1]["_token_signals_dropped"]; + assert_eq!( + dropped["completion_token_ids"], per_turn as u64, + "a dropped signal must name itself and its length: {dropped}" + ); + assert_eq!( + t["turns"][1]["_token_signals_dropped"]["response_mask"], per_turn as u64, + "every refused signal on the turn must name itself" ); - assert_eq!(turns[0]["decisions"].as_array().unwrap().len(), 1); } + /// Misaligned completion signals are discarded whole — the same loss a + /// budget drop is — so the row is degraded, not merely annotated. #[test] - fn build_trajectory_no_spans_emits_empty_decisions() { - let fields = json!({ "user_message": "", "has_result": false }); - let t = build_trajectory( - "trj-0", - "ss-0", - "aj-0", - "Failed", - &fields, - "", - &entity_state_with_events(), + fn build_trajectory_marks_misaligned_signals_as_degraded() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}], + "completion_token_ids":[7, 8, 9], + "response_mask":[1, 1], + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!(t["turns"][0]["_token_signals_misaligned"].is_object()); + assert!( + t["turns"][0].get("completion_token_ids").is_none(), + "the misaligned set is discarded whole" + ); + assert!( + degradations(&t).contains(&"token_signals_dropped".to_string()), + "the discard must reach the Session status, not only the document: {:?}", + t["metadata"]["tags"] ); - assert_eq!(t["metadata"]["outcome"], "failure"); - assert_eq!(t["turns"][0]["decisions"].as_array().unwrap().len(), 0); } + /// The tag says a consumer can read token signals off this row. A turn whose + /// signals were all dropped carries a record of the drop and no signal, so + /// tagging it would send a consumer looking for data that is not there. #[test] - fn build_trajectory_cancelled_is_partial_success() { - let fields = json!({ "user_message": "", "has_result": false }); - let t = build_trajectory( - "trj-c", - "ss-c", - "aj-c", - "Cancelled", - &fields, - "", - &entity_state_with_events(), + fn build_trajectory_does_not_advertise_signals_it_dropped() { + let oversized = MAX_TOKEN_SIGNAL_BYTES; + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}], + "completion_token_ids": vec![123456_u64; oversized], + "response_mask": vec![1_u64; oversized], + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!( + t["turns"][0].get("completion_token_ids").is_none(), + "the signal must not have been written" + ); + assert!( + t["turns"][0]["_token_signals_dropped"].is_object(), + "the drop itself still has to be recorded on the turn" + ); + assert!( + degradations(&t).contains(&"token_signals_dropped".to_string()), + "a row built without signals it was offered is degraded: {:?}", + t["metadata"]["tags"] ); - assert_eq!(t["metadata"]["outcome"], "partial_success"); } + /// When the entry's extras were themselves cut to fit the field ceiling, + /// which members went is no longer knowable — only how many. The turn's + /// record is short either way, and the row must say so. #[test] - fn build_trajectory_includes_system_message_when_present() { - let fields = - json!({ "user_message": "", "system_prompt": "you are a helpful agent" }); - let t = build_trajectory( - "trj-s", - "ss-s", - "aj-s", - "Completed", - &fields, - "", - &entity_state_with_events(), + fn build_trajectory_marks_turn_extras_cut_to_fit_as_degraded() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}], + "_extra_json_dropped_members": 12 + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!(t["turns"][0]["_turn_extras_dropped_members"], 12); + assert!( + degradations(&t).contains(&"token_signals_dropped".to_string()), + "extras cut to fit take turn facts with them, so the row is short: {:?}", + t["metadata"]["tags"] + ); + } + + /// The SessionEntry writer refuses signals that would push the entry over + /// its own ceiling and leaves `_dropped_bytes` behind. Without + /// carrying that forward, a turn whose signals were all refused at capture + /// looks exactly like a provider that never sent any. + #[test] + fn build_trajectory_carries_capture_stage_signal_drops() { + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}], + "completion_token_ids_dropped_bytes": 40_000, + "logprobs_dropped_bytes": 52_000 + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!( + t["turns"][0]["_token_signals_dropped"]["completion_token_ids"], + 40_000 + ); + assert_eq!( + t["turns"][0]["_token_signals_dropped"]["logprobs"], 52_000, + "a refusal at capture must still be legible on the turn" ); - // OTSSystemMessage has { content, timestamp } only — no `role` field. - assert_eq!(t["system_message"]["content"], "you are a helpful agent"); assert!( - t["system_message"]["timestamp"] - .as_str() - .is_some_and(|s| !s.is_empty()), - "system_message.timestamp must be non-empty" + degradations(&t).contains(&"token_signals_dropped".to_string()), + "and must reach the Session status: {:?}", + t["metadata"]["tags"] ); - assert!(t["system_message"].get("role").is_none(), "role must be omitted"); } + /// The JCS contract fields ride natively now: the pinned `temper-ots` + /// models every one, so a consumer that deserializes a stored row and + /// writes it back keeps them, and the carriers that stood in for them are + /// gone. + /// + /// This is what those carriers were removed against, so it fails if either + /// half comes back. Each field must be emitted, survive the round trip with + /// its value intact, and be mirrored nowhere: no `harness:` / + /// `spec_version:` / `token_signals:present` tag, no `context.entities` + /// inventory. #[test] - fn build_trajectory_skips_system_message_when_empty() { - let fields = json!({ "user_message": "", "system_prompt": "" }); - let t = build_trajectory( - "trj-s2", - "ss-s2", - "aj-s2", - "Completed", - &fields, - "", - &entity_state_with_events(), + fn kernel_round_trip_keeps_the_jcs_contract_fields() { + use temper_ots::models::OTSTrajectory; + + let fields = two_turn_fields(); + let state = entity_state_with_events(); + let jsonl = token_signal_session_jsonl(2); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.read\",\"result\":\"ok\",\"duration_ms\":1,\"is_error\":false}\n"; + let emitted = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + let trajectory: OTSTrajectory = + serde_json::from_value(emitted.clone()).expect("document deserializes"); + let round_tripped = serde_json::to_value(&trajectory).expect("re-serializes"); + + let contract_fields: [(&str, &Value, &Value); 7] = [ + ( + "metadata.harness", + &emitted["metadata"]["harness"], + &round_tripped["metadata"]["harness"], + ), + ( + "metadata.spec_version", + &emitted["metadata"]["spec_version"], + &round_tripped["metadata"]["spec_version"], + ), + ( + "turns[].prompt_token_ids", + &emitted["turns"][0]["prompt_token_ids"], + &round_tripped["turns"][0]["prompt_token_ids"], + ), + ( + "turns[].completion_token_ids", + &emitted["turns"][0]["completion_token_ids"], + &round_tripped["turns"][0]["completion_token_ids"], + ), + ( + "turns[].response_mask", + &emitted["turns"][0]["response_mask"], + &round_tripped["turns"][0]["response_mask"], + ), + ( + "turns[].logprobs", + &emitted["turns"][0]["logprobs"], + &round_tripped["turns"][0]["logprobs"], + ), + ( + "turns[].decisions[].cause_id", + &emitted["turns"][0]["decisions"][0]["cause_id"], + &round_tripped["turns"][0]["decisions"][0]["cause_id"], + ), + ]; + for (path, before, after) in contract_fields { + assert!( + !before.is_null(), + "the emitter stopped producing {path}; this test only means \ + something while every contract field is emitted" + ); + assert_eq!( + before, after, + "{path} did not survive the kernel round trip. Either the pin \ + rolled back to a revision that does not model it, or the field \ + drifted — do not answer that by reintroducing a carrier" + ); + } + + // Typed access, not just JSON shape: the fields are on the structs. + assert_eq!(trajectory.metadata.harness.as_deref(), Some(HARNESS)); + assert_eq!( + trajectory.metadata.spec_version.as_deref(), + Some("paw-agent@0.1.0") + ); + assert_eq!( + trajectory.turns[0].decisions[0].cause_id.as_deref(), + Some("tc-1") + ); + + // The carriers are gone and must stay gone. + for tag in trajectory.metadata.tags.iter() { + assert!( + !tag.starts_with("harness:") + && !tag.starts_with("spec_version:") + && !tag.starts_with("token_signals:"), + "a provenance mirror came back as {tag:?}; the kernel models \ + these directly now" + ); + } + assert!( + round_tripped["context"].get("entities").is_none(), + "the turn_token_signals inventory came back: {}", + round_tripped["context"] ); - assert!(t.get("system_message").is_none()); } + /// Serde ignores unknown fields, so a round trip says nothing about a field + /// the kernel does not model. One is left by design — the POST handler reads + /// `metadata.trajectory_id` before any struct is involved — and this pins + /// that it is the only one, so a new extension cannot appear unnoticed. #[test] - fn build_trajectory_populates_timestamp_bookends_from_events() { - let fields = json!({ "user_message": "x", "has_result": true }); - let entity_state = json!({ - "events": [ - { "action": "Created", "timestamp": "2026-01-01T00:00:00Z" }, - { "action": "Cancel", "timestamp": "2026-01-01T00:00:05Z" }, - ] + fn kernel_round_trip_drops_exactly_the_unmodeled_extensions() { + use temper_ots::models::OTSTrajectory; + + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"tool_use","id":"tc-1","name":"temper.read","input":{}}], + "prompt_token_ids":[1,2], + "completion_token_ids":[3,4], + "response_mask":[1,1], + "logprobs":[-0.1,-0.2] + }), + json!({ + "id":"t-2","parentId":"a-1","type":"message","role":"user", + "content":[{"type":"tool_result","tool_use_id":"tc-1","content":"ok"}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "t-2", "has_result": true }); + let state = entity_state_with_events(); + let emitted = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let trajectory: OTSTrajectory = + serde_json::from_value(emitted.clone()).expect("document deserializes"); + let round_tripped = serde_json::to_value(&trajectory).expect("re-serializes"); + + let mut dropped: Vec = Vec::new(); + collect_dropped_paths(&emitted, &round_tripped, "", &mut dropped); + let dropped: BTreeSet = dropped + .into_iter() + // The `_`-prefixed fields are declared non-standard by ADR-0035 and + // are expected to be kernel-invisible; the ones without the prefix + // read like OTS fields and are the ones worth pinning. + .filter(|path| !path.split('.').any(|segment| segment.starts_with('_'))) + .collect(); + let expected: BTreeSet = KERNEL_UNMODELED_FIELDS + .iter() + .map(|field| (*field).to_string()) + .collect(); + + assert_eq!( + dropped, expected, + "the set of emitter fields the pinned kernel drops has changed; \ + update KERNEL_UNMODELED_FIELDS and ADR-0035 deliberately" + ); + } + + /// Rows written before these fields existed must keep deserializing — the + /// additions are additive, and an evaluation worker reads old rows too. + #[test] + fn rows_without_the_new_fields_still_deserialize() { + use temper_ots::models::{OTSTrajectory, OutcomeType}; + + let old_row = json!({ + "trajectory_id": "trj-old", + "version": "0.1.0", + "metadata": { + "trajectory_id": "trj-old", + "task_description": "an older run", + "domain": "temperpaw-agent", + "timestamp_start": "2026-01-01T00:00:00Z", + "timestamp_end": "2026-01-01T00:00:05Z", + "agent_id": "aj-old", + "framework": "temperpaw", + "environment": "production", + "outcome": "success", + "tags": ["claude-sonnet-4-6"], + }, + "turns": [{ + "turn_id": 1, + "span_id": "ss-old:a-1", + "timestamp": "2026-01-01T00:00:01Z", + "error": false, + "messages": [], + "decisions": [{ + "decision_id": "tc-old", + "decision_type": "tool_selection", + "choice": { "action": "temper.read" }, + "consequence": { "success": true, "result_summary": "ok" }, + }], + }], }); - let t = build_trajectory( - "trj-ts", - "ss-ts", - "aj-ts", - "Completed", - &fields, - "", - &entity_state, + + let trajectory: OTSTrajectory = serde_json::from_value(old_row) + .expect("a pre-ARN-109 row must still deserialize as an OTSTrajectory"); + assert_eq!(trajectory.metadata.outcome, OutcomeType::Success); + assert_eq!(trajectory.turns[0].decisions[0].decision_id, "tc-old"); + assert_eq!(trajectory.metadata.tags, vec!["claude-sonnet-4-6"]); + } + + fn is_empty_collection(value: &Value) -> bool { + match value { + Value::Array(items) => items.is_empty(), + Value::Object(fields) => fields.is_empty(), + Value::Null => true, + _ => false, + } + } + + /// Record every leaf path present in `emitted` but absent from + /// `round_tripped`, using the field paths `KERNEL_UNMODELED_FIELDS` names. + fn collect_dropped_paths( + emitted: &Value, + round_tripped: &Value, + path: &str, + dropped: &mut Vec, + ) { + match emitted { + Value::Object(fields) => { + for (key, value) in fields { + let child_path = if path.is_empty() { + key.clone() + } else { + format!("{path}.{key}") + }; + match round_tripped.get(key) { + Some(other) => { + collect_dropped_paths(value, other, &child_path, dropped) + } + // An empty collection the kernel omits on write carries + // no data, so its absence is formatting, not loss. + None if is_empty_collection(value) => {} + None => dropped.push(child_path), + } + } + } + Value::Array(items) => { + let others = round_tripped.as_array(); + for (index, item) in items.iter().enumerate() { + let Some(other) = others.and_then(|others| others.get(index)) else { + continue; + }; + collect_dropped_paths(item, other, &format!("{path}[]"), dropped); + } + } + _ => {} + } + } + + #[test] + fn build_trajectory_bounds_oversized_tool_arguments() { + let big_argument = "z".repeat(MAX_ARGUMENTS_CHARS * 2); + let jsonl = [ + json!({"id":"u-1","parentId":null,"type":"message","role":"user","content":"go"}), + json!({ + "id":"a-1","parentId":"u-1","type":"message","role":"assistant", + "content":[{"type":"tool_use","id":"tc-big","name":"temper.write","input":{"body":big_argument}}] + }), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "session_leaf_id": "a-1" }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + let arguments = &t["turns"][0]["decisions"][0]["choice"]["arguments"]; + assert_eq!(arguments["_truncated"], true); + assert!( + arguments["_preview"] + .as_str() + .unwrap() + .chars() + .count() + <= MAX_ARGUMENTS_CHARS + ); + } + + /// A transcript that is not there produces a spans-only document. Storing it + /// as if it were complete is the failure this marks: the row is written once + /// and the session is marked emitted, so nothing downstream ever revisits it. + #[test] + fn build_trajectory_marks_an_absent_transcript_as_degraded() { + let fields = json!({ "has_result": true, "tool_spans_file_id": "file-spans-1" }); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"result\":\"ok\",\"duration_ms\":3,\"is_error\":false}\n"; + let state = entity_state_with_events(); + + for (presence, expected) in [ + (TranscriptPresence::MissingFile, "transcript_missing_file"), + (TranscriptPresence::EmptyFile, "transcript_empty_file"), + ( + TranscriptPresence::PendingFirstTurn, + "transcript_pending_first_turn", + ), + (TranscriptPresence::NoEntries, "transcript_no_entries"), + (TranscriptPresence::Undeclared, "transcript_undeclared"), + ] { + let mut input = inputs(&fields, "", spans, &state, "Completed"); + input.transcript = presence; + let t = build_trajectory(&input); + + assert_eq!( + degradations(&t), + vec![expected.to_string()], + "a {} transcript must be reported as degraded", + presence.as_str() + ); + assert_eq!(t["_transcript"]["present"], false); + assert_eq!(t["_transcript"]["reasons"], json!([presence.as_str()])); + } + } + + /// A transcript that arrived but does not parse is missing history just as + /// surely as one that never arrived. Judging completeness by whether bytes + /// showed up would store a corrupted file as a complete record — the same + /// false-complete row an absent transcript used to produce, reached through + /// corruption instead of a 404. + #[test] + fn build_trajectory_marks_an_unparseable_transcript_as_degraded() { + let mut lines: Vec = two_turn_session_jsonl() + .lines() + .map(str::to_string) + .collect(); + lines.push("{\"id\":\"a-4\",\"parentId\":\"a-3\",\"type\"".to_string()); // write cut mid-line + let jsonl = lines.join("\n"); + let fields = two_turn_fields(); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!(degradations(&t), vec!["transcript_unparseable".to_string()]); + assert_eq!(t["_transcript"]["present"], true); + assert_eq!(t["_transcript"]["unparsed_lines"], 1); + assert_eq!( + t["turns"].as_array().unwrap().len(), + 2, + "the readable turns are still kept — one bad line must not cost the trajectory" ); + } + + /// The recorded leaf is the session's own claim about where its history + /// ends. When it does not resolve, the fallback chain is an older one — the + /// newest turns are exactly what is missing — and the row must not pass as + /// the whole session. + #[test] + fn build_trajectory_marks_an_unresolved_leaf_as_degraded() { + let mut fields = two_turn_fields(); + fields["session_leaf_id"] = json!("a-5-never-written"); + let jsonl = two_turn_session_jsonl(); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + assert_eq!( - t["metadata"]["timestamp_start"], "2026-01-01T00:00:00Z", - "first event timestamp must map to metadata.timestamp_start" + degradations(&t), + vec!["transcript_leaf_unresolved".to_string()] ); + assert_eq!(t["_transcript"]["present"], true); assert_eq!( - t["metadata"]["timestamp_end"], "2026-01-01T00:00:05Z", - "last event timestamp must map to metadata.timestamp_end" + t["turns"].as_array().unwrap().len(), + 2, + "the recoverable turns are still emitted" ); } + /// A transcript whose entries yield no turn at all produces the same + /// synthetic single-turn document as an empty one, and must be labelled the + /// same way rather than passing as a session that genuinely did nothing. #[test] - fn build_trajectory_missing_events_emits_epoch_fallback() { - let fields = json!({ "user_message": "x" }); - let t = build_trajectory( - "trj-noev", - "ss-noev", - "aj-noev", - "Completed", - &fields, - "", - &json!({}), + fn build_trajectory_marks_a_transcript_without_turns_as_degraded() { + let jsonl = json!({"id":"h-ss-1","parentId":null,"type":"header","tokens":0}).to_string(); + let fields = json!({ "session_leaf_id": "h-ss-1", "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!( + degradations(&t).contains(&"transcript_no_turns".to_string()), + "tags: {:?}", + t["metadata"]["tags"] ); - assert_eq!(t["metadata"]["timestamp_start"], "1970-01-01T00:00:00Z"); - assert_eq!(t["metadata"]["timestamp_end"], "1970-01-01T00:00:00Z"); + assert_eq!(t["turns"].as_array().unwrap().len(), 1); } - fn entity_state_with_events() -> Value { - json!({ - "events": [ - { "action": "Created", "timestamp": "2026-01-01T00:00:00Z" }, - { "action": "Cancel", "timestamp": "2026-01-01T00:00:01Z" }, - ] - }) + /// Span lines are skipped when they do not parse, so a partially written + /// append leaves tool calls with no evidence and nothing saying so. + #[test] + fn build_trajectory_marks_unparseable_tool_spans_as_degraded() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = concat!( + "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"result\":\"ok\",\"duration_ms\":3,\"is_error\":false}\n", + "{\"tool_call_id\":\"tc-2\",\"tool_name\":\"temper.re\n" + ); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + assert_eq!( + degradations(&t), + vec!["tool_spans_unparseable".to_string()] + ); + assert_eq!(t["_tool_spans_unparsed_lines"], 1); + } + + /// A complete run must not be labelled degraded — the marker is only useful + /// if it means something. + #[test] + fn build_trajectory_reports_no_degradation_for_a_complete_run() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let spans = "{\"tool_call_id\":\"tc-1\",\"tool_name\":\"temper.bash\",\"result\":\"ok\",\"duration_ms\":3,\"is_error\":false}\n"; + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, spans, &state, "Completed")); + + assert!(degradations(&t).is_empty(), "tags: {:?}", t["metadata"]["tags"]); + assert!(t.get("_transcript").is_none()); + assert!(t.get("_tool_spans_missing").is_none()); + } + + /// A declared span file that 404s is missing evidence, not an absence of + /// tool calls, and a truncated span document is missing tool timings. + #[test] + fn build_trajectory_marks_missing_and_truncated_tool_spans() { + let fields = two_turn_fields(); + let jsonl = two_turn_session_jsonl(); + let state = entity_state_with_events(); + + let mut input = inputs(&fields, &jsonl, "", &state, "Completed"); + input.tool_spans_missing = true; + let t = build_trajectory(&input); + assert_eq!(degradations(&t), vec!["tool_spans_missing_file".to_string()]); + assert_eq!(t["_tool_spans_missing"], true); + + let sealed = format!( + "{}\n", + json!({"tool_name": TOOL_SPANS_TRUNCATED_MARKER, "tool_call_id":"", "result":"", "duration_ms":0, "is_error":false}) + ); + let t = build_trajectory(&inputs(&fields, &jsonl, &sealed, &state, "Completed")); + assert_eq!(t["_tool_spans_truncated"], true); + assert_eq!(degradations(&t), vec!["tool_spans_truncated".to_string()]); + } + + /// A span write that failed leaves `tool_spans_file_id` empty, which reads + /// exactly like a session that called no tools. The session records the + /// failure so the two can be told apart. + #[test] + fn build_trajectory_marks_a_failed_span_write_as_degraded() { + let mut fields = two_turn_fields(); + fields["tool_spans_file_id"] = json!(""); + fields["tool_spans_write_failed"] = json!("true"); + let jsonl = two_turn_session_jsonl(); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert_eq!( + degradations(&t), + vec!["tool_spans_write_failed".to_string()] + ); + assert_eq!(t["_tool_spans_write_failed"], true); + } + + /// Raw file order is not a walk. When no leaf resolves — recorded or not — + /// the parent structure is unusable and the order is a guess, so the + /// document must not present it as the session's shape. + #[test] + fn build_trajectory_marks_raw_order_fallback_as_degraded() { + // Every entry points at a parent that is not in the transcript, so no + // leaf walks, and no leaf was recorded either. + let jsonl = [ + json!({"id":"u-1","parentId":"missing-a","type":"message","role":"user","content":"go"}), + json!({"id":"a-1","parentId":"missing-b","type":"message","role":"assistant","content":"ok"}), + ] + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"); + let fields = json!({ "has_result": true }); + let state = entity_state_with_events(); + let t = build_trajectory(&inputs(&fields, &jsonl, "", &state, "Completed")); + + assert!( + degradations(&t).contains(&"transcript_leaf_unresolved".to_string()), + "tags: {:?}", + t["metadata"]["tags"] + ); + + let resolved = resolve_chain(&parse_session_entries(&jsonl), ""); + assert!(!resolved.from_recorded_leaf); + } + + /// The completeness marker is the last thing that may be lost on a round + /// trip: without it a partial record reads as a whole one. + #[test] + fn degradation_markers_survive_the_kernel_round_trip() { + use temper_ots::models::OTSTrajectory; + + let fields = json!({ "has_result": false }); + let state = entity_state_with_events(); + let mut input = inputs(&fields, "", "", &state, "Failed"); + input.transcript = TranscriptPresence::MissingFile; + input.tool_spans_missing = true; + let document = build_trajectory(&input); + + let trajectory: OTSTrajectory = + serde_json::from_value(document).expect("document deserializes"); + let round_tripped = serde_json::to_value(&trajectory).expect("re-serializes"); + + assert_eq!( + degradations(&round_tripped), + vec![ + "transcript_missing_file".to_string(), + "tool_spans_missing_file".to_string() + ], + "tags: {:?}", + trajectory.metadata.tags + ); } } diff --git a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml index 8ae91e35f..f4a743be5 100644 --- a/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml +++ b/os-apps/paw-agent/wasm/guest_observability_probe/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml index 453b55583..b90eb949c 100644 --- a/os-apps/paw-agent/wasm/monty_repl/Cargo.toml +++ b/os-apps/paw-agent/wasm/monty_repl/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } tool-catalog = { path = "../tool-catalog" } diff --git a/os-apps/paw-agent/wasm/monty_repl/src/lib.rs b/os-apps/paw-agent/wasm/monty_repl/src/lib.rs index 058b2e550..69d6c0e68 100644 --- a/os-apps/paw-agent/wasm/monty_repl/src/lib.rs +++ b/os-apps/paw-agent/wasm/monty_repl/src/lib.rs @@ -1035,10 +1035,17 @@ pub extern "C" fn run(_ctx_ptr: i32, _ctx_len: i32) -> i32 { params["tool_spans_file_id"] = json!(id); } Ok(_) => {} - Err(e) => ctx.log( - "warn", - &format!("monty_repl: tool_spans append failed: {e}"), - ), + Err(e) => { + // The tool calls happened; only the record of them is gone. + // An empty tool_spans_file_id reads exactly like a session + // that called no tools, so the loss is recorded on the + // entity and the emitter degrades the trajectory for it. + params["tool_spans_write_failed"] = json!("true"); + ctx.log( + "warn", + &format!("monty_repl: tool_spans append failed: {e}"), + ); + } } } else if !tool_span_events.is_empty() { ctx.log( @@ -1229,11 +1236,24 @@ fn normal_repl_state_max_bytes(ctx: &Context) -> usize { } } +/// Whether to persist this batch's tool spans to the session's span file. +/// +/// Defaults to ON. The spans are the only record of tool-call wall-clock time, +/// and the OTS emitter reads them to complete each decision — a missing config +/// key silently emptying every stored trajectory (exactly what happened in +/// production, ARN-109) is a worse failure than the write cost, which is +/// bounded by `session::encode_tool_spans_jsonl`. Set the key to a false-y +/// value to opt a deployment out. fn persist_tool_spans_file(ctx: &Context) -> bool { ctx.config .get("persist_tool_spans_file") - .map(|value| matches!(value.as_str(), "1" | "true" | "yes" | "on")) - .unwrap_or(false) + .map(|value| { + !matches!( + value.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "no" | "off" + ) + }) + .unwrap_or(true) } fn attach_llmobs_tool_spans(params: &mut Value, tool_span_events: &[Value]) { diff --git a/os-apps/paw-agent/wasm/monty_repl/src/session.rs b/os-apps/paw-agent/wasm/monty_repl/src/session.rs index 226340fa9..d8b2fe2e3 100644 --- a/os-apps/paw-agent/wasm/monty_repl/src/session.rs +++ b/os-apps/paw-agent/wasm/monty_repl/src/session.rs @@ -64,6 +64,7 @@ pub fn persist_results( "user", &tool_results_value, tokens_est, + None, )?; // create_session_entry now does a read-back verify, so reaching // here means the row is durably visible. @@ -367,11 +368,80 @@ fn progress_action_dispatch_enabled(ctx: &Context, fields: &Value, key: &str) -> .unwrap_or(false) } +/// Largest `result` kept in a persisted tool span. The OTS decision only ever +/// shows the first 500 characters; the full result already lives in the +/// tool_result session entry, so a longer copy here buys nothing and the span +/// file is rewritten in full on every tool batch. +const TOOL_SPAN_RESULT_MAX_CHARS: usize = 600; +/// Largest `arguments` payload kept in a persisted tool span. +const TOOL_SPAN_ARGUMENTS_MAX_CHARS: usize = 2_000; +/// Ceiling on the whole tool-span document. Past this the session stops +/// appending rather than paying an unbounded read-modify-write per tool batch. +const TOOL_SPANS_FILE_MAX_BYTES: usize = 262_144; +/// Reserved tool name marking a span document that hit the ceiling. The OTS +/// emitter recognizes it and records the truncation instead of turning it into +/// a decision. +pub const TOOL_SPANS_TRUNCATED_MARKER: &str = "_tool_spans_truncated"; +/// The marker line, written once when the document seals. +const TOOL_SPANS_TRUNCATED_LINE: &str = "{\"tool_call_id\":\"\",\"tool_name\":\"_tool_spans_truncated\",\"result\":\"tool span file size ceiling reached\",\"duration_ms\":0,\"is_error\":false}\n"; + +/// True once the document has been sealed by a truncation marker. +/// +/// Only the last non-empty line is examined, and only its `tool_name` field: +/// the marker is a reserved tool name, so a substring search would also fire on +/// a tool result that merely quotes it. Byte slicing is avoided outright — a +/// result carrying CJK or emoji puts arbitrary offsets inside a character. +fn tool_spans_document_sealed(document: &str) -> bool { + document + .lines() + .rev() + .find(|line| !line.trim().is_empty()) + .is_some_and(is_tool_spans_truncation_line) +} + +/// True when a JSONL line is the reserved truncation marker record. +fn is_tool_spans_truncation_line(line: &str) -> bool { + serde_json::from_str::(line.trim()) + .ok() + .as_ref() + .and_then(|value| value.get("tool_name")) + .and_then(Value::as_str) + == Some(TOOL_SPANS_TRUNCATED_MARKER) +} + +fn truncate_span_chars(value: &str, max_chars: usize) -> String { + let total = value.chars().count(); + if total <= max_chars { + return value.to_string(); + } + let mut out: String = value.chars().take(max_chars).collect(); + out.push_str(&format!("...[truncated {} of {total} chars]", total - max_chars)); + out +} + +/// Shrink a tool-span event to what the trajectory emitter actually consumes. +pub fn compact_tool_span(event: &Value) -> Value { + let mut compacted = event.clone(); + if let Some(object) = compacted.as_object_mut() { + if let Some(Value::String(result)) = object.get("result") { + let bounded = truncate_span_chars(result, TOOL_SPAN_RESULT_MAX_CHARS); + object.insert("result".to_string(), json!(bounded)); + } + if let Some(Value::String(arguments)) = object.get("arguments") { + let bounded = truncate_span_chars(arguments, TOOL_SPAN_ARGUMENTS_MAX_CHARS); + object.insert("arguments".to_string(), json!(bounded)); + } + } + compacted +} + /// Encode a tool-span JSONL document by appending new events to the existing content. /// /// Each event in `new_events` is serialized as a single JSON object on its own line, /// separated by '\n'. The returned string always ends with '\n' so that subsequent -/// appends stay line-delimited. +/// appends stay line-delimited. Events are compacted first, and appends stop once +/// the document reaches `TOOL_SPANS_FILE_MAX_BYTES` so a long session cannot turn +/// the per-batch rewrite into unbounded traffic. pub fn encode_tool_spans_jsonl(existing: &str, new_events: &[Value]) -> String { let mut out = String::with_capacity(existing.len() + new_events.len() * 128); if !existing.is_empty() { @@ -380,8 +450,28 @@ pub fn encode_tool_spans_jsonl(existing: &str, new_events: &[Value]) -> String { out.push('\n'); } } + if tool_spans_document_sealed(&out) { + return out; + } + if out.len() >= TOOL_SPANS_FILE_MAX_BYTES { + // At the ceiling with no seal on it: a document written before this + // ceiling existed, or a batch that landed byte-exact on it. The spans + // in hand are being dropped, so the document has to say so — otherwise + // it parses clean and the trajectory built from it reads as holding + // every tool call the session made. Sealing costs one line over the + // ceiling, which is bounded; silence costs the record. + if !new_events.is_empty() { + out.push_str(TOOL_SPANS_TRUNCATED_LINE); + } + return out; + } for event in new_events { - let line = serde_json::to_string(event).unwrap_or_else(|_| "{}".to_string()); + let line = serde_json::to_string(&compact_tool_span(event)) + .unwrap_or_else(|_| "{}".to_string()); + if out.len() + line.len() + 1 > TOOL_SPANS_FILE_MAX_BYTES { + out.push_str(TOOL_SPANS_TRUNCATED_LINE); + break; + } out.push_str(&line); out.push('\n'); } @@ -887,4 +977,174 @@ mod tests { let out = encode_tool_spans_jsonl(existing, &[]); assert_eq!(out, existing); } + + #[test] + fn compact_tool_span_bounds_result_and_arguments() { + let event = json!({ + "tool_call_id": "a", + "tool_name": "temper.bash", + "arguments": "x".repeat(TOOL_SPAN_ARGUMENTS_MAX_CHARS + 500), + "result": "y".repeat(TOOL_SPAN_RESULT_MAX_CHARS + 5_000), + "duration_ms": 12, + "is_error": false, + }); + let compacted = compact_tool_span(&event); + assert_eq!(compacted["tool_call_id"], "a"); + assert_eq!(compacted["duration_ms"], 12); + let result = compacted["result"].as_str().unwrap(); + assert!(result.starts_with(&"y".repeat(TOOL_SPAN_RESULT_MAX_CHARS))); + assert!(result.contains("truncated 5000 of")); + let arguments = compacted["arguments"].as_str().unwrap(); + assert!(arguments.contains("truncated 500 of")); + } + + #[test] + fn compact_tool_span_leaves_small_events_untouched() { + let event = json!({ + "tool_call_id": "a", + "tool_name": "read", + "arguments": "{\"path\":\"/tmp/x\"}", + "result": "ok", + "duration_ms": 3, + "is_error": false, + }); + assert_eq!(compact_tool_span(&event), event); + } + + #[test] + fn encode_tool_spans_jsonl_stops_at_the_size_ceiling() { + let event = json!({ + "tool_call_id": "a", + "tool_name": "temper.bash", + "arguments": "z".repeat(TOOL_SPAN_ARGUMENTS_MAX_CHARS), + "result": "w".repeat(TOOL_SPAN_RESULT_MAX_CHARS), + "duration_ms": 1, + "is_error": false, + }); + let events: Vec = std::iter::repeat_n(event, 400).collect(); + let out = encode_tool_spans_jsonl("", &events); + assert!( + out.len() <= TOOL_SPANS_FILE_MAX_BYTES, + "span file must stay bounded, got {} bytes", + out.len() + ); + assert!(out.contains("_tool_spans_truncated")); + // Every line still parses — a truncated document must stay readable. + for line in out.split_terminator('\n') { + serde_json::from_str::(line).expect("each line must be valid JSON"); + } + } + + #[test] + fn encode_tool_spans_jsonl_seals_a_full_document_it_refuses_to_grow() { + let existing = format!("{}\n", "x".repeat(TOOL_SPANS_FILE_MAX_BYTES)); + let out = encode_tool_spans_jsonl(&existing, &[json!({"tool_call_id": "next"})]); + + assert!( + out.starts_with(&existing), + "a full document must not grow with new spans" + ); + assert!( + !out.contains("\"tool_call_id\":\"next\""), + "the refused span must not be written" + ); + assert!( + tool_spans_document_sealed(&out), + "a document that is dropping spans must say so; a clean parse of it \ + otherwise reads as a complete record of the session's tool calls" + ); + + // Sealing is idempotent — later batches find the seal and stop. + let again = encode_tool_spans_jsonl(&out, &[json!({"tool_call_id": "later"})]); + assert_eq!(again, out, "an already sealed document must not grow"); + assert_eq!( + again.matches(TOOL_SPANS_TRUNCATED_MARKER).count(), + 1, + "the seal is written once" + ); + } + + #[test] + fn encode_tool_spans_jsonl_does_not_seal_a_full_document_with_nothing_to_add() { + let existing = format!("{}\n", "x".repeat(TOOL_SPANS_FILE_MAX_BYTES)); + let out = encode_tool_spans_jsonl(&existing, &[]); + assert_eq!( + out, existing, + "no spans were dropped, so nothing was lost and nothing is marked" + ); + } + + #[test] + fn encode_tool_spans_jsonl_seals_the_document_once() { + let event = json!({ + "tool_call_id": "a", + "tool_name": "temper.bash", + "arguments": "z".repeat(TOOL_SPAN_ARGUMENTS_MAX_CHARS), + "result": "w".repeat(TOOL_SPAN_RESULT_MAX_CHARS), + "duration_ms": 1, + "is_error": false, + }); + let events: Vec = std::iter::repeat_n(event.clone(), 400).collect(); + let sealed = encode_tool_spans_jsonl("", &events); + let markers = sealed.matches(TOOL_SPANS_TRUNCATED_MARKER).count(); + assert_eq!(markers, 1); + + // A later batch must not stack a second marker onto a sealed document. + let again = encode_tool_spans_jsonl(&sealed, &[event]); + assert_eq!(again, sealed); + assert_eq!(again.matches(TOOL_SPANS_TRUNCATED_MARKER).count(), 1); + } + + /// Tool output is arbitrary UTF-8. The seal check used to slice the document + /// at a byte offset, which traps the guest mid-character on any multibyte + /// tail — after the tools already ran, so the result callback never fires. + #[test] + fn encode_tool_spans_jsonl_survives_multibyte_tool_output() { + let existing = format!( + "{}\n", + json!({ + "tool_call_id": "a", + "tool_name": "temper.bash", + "result": "日本語のテキスト🎌".repeat(20), + "duration_ms": 1, + "is_error": false, + }) + ); + let out = encode_tool_spans_jsonl(&existing, &[json!({"tool_call_id": "b"})]); + assert!(out.starts_with(&existing)); + assert!(out.contains("\"tool_call_id\":\"b\"")); + } + + /// A tool that reads or greps this source returns the marker literal in its + /// own output. That must not look like a sealed document. + #[test] + fn encode_tool_spans_jsonl_ignores_a_quoted_marker_in_tool_output() { + let existing = format!( + "{}\n", + json!({ + "tool_call_id": "a", + "tool_name": "temper.read", + "result": format!("const MARKER = \"{TOOL_SPANS_TRUNCATED_MARKER}\";"), + "duration_ms": 1, + "is_error": false, + }) + ); + let out = encode_tool_spans_jsonl(&existing, &[json!({"tool_call_id": "b"})]); + assert!( + out.contains("\"tool_call_id\":\"b\""), + "a quoted marker must not seal the document" + ); + } + + #[test] + fn tool_spans_document_sealed_reads_the_marker_record_only() { + assert!(!tool_spans_document_sealed("")); + assert!(tool_spans_document_sealed(TOOL_SPANS_TRUNCATED_LINE)); + assert!( + !tool_spans_document_sealed( + "{\"tool_call_id\":\"a\",\"tool_name\":\"read\",\"result\":\"_tool_spans_truncated\"}\n" + ), + "the marker is a reserved tool_name, not a substring of any field" + ); + } } diff --git a/os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs b/os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs index 1c0e2cfcc..e720c9e6e 100644 --- a/os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs +++ b/os-apps/paw-agent/wasm/openai-chat-wire/src/lib.rs @@ -59,6 +59,124 @@ pub struct ParsedChatCompletion { pub response_bytes: usize, pub semantic_deltas: Vec, pub completed: bool, + /// Token-level RL signals the serving stack streamed alongside the text + /// (`logprobs`, `prompt_token_ids`, `completion_token_ids`, + /// `response_mask`). `None` unless the server actually sent them — the + /// caller never requests a second round trip to obtain them. + pub token_signals: Option, +} + +/// Token-level RL signal field names, in the shape OTS turns use. +pub const TOKEN_SIGNAL_FIELDS: &[&str] = &[ + "prompt_token_ids", + "completion_token_ids", + "response_mask", + "logprobs", +]; + +/// Collapse one stream event's two possible signal homes into a single source. +/// +/// Completion-side signals accumulate across events, so a signal taken twice +/// from one event is stored twice — and when only one signal is present nothing +/// downstream can detect it, because there is no second array to disagree on +/// length. Every accumulator routes an event through this before merging, +/// whatever its wire shape. +/// +/// `content` is the level carrying the model's output for that event: the +/// choice in a chat-completions chunk, the response in a Responses +/// `response.completed`. `usage` is the token-accounting object beside it. +/// Where both carry the same field the content level wins — `usage` repeating a +/// signal is a server quirk, not a second measurement. `None` when neither +/// level carries any signal field. +pub fn event_token_signals(usage: Option<&Value>, content: Option<&Value>) -> Option { + let mut source = Map::new(); + for field in TOKEN_SIGNAL_FIELDS { + let value = content + .and_then(|content| content.get(*field)) + .or_else(|| usage.and_then(|usage| usage.get(*field))); + if let Some(value) = value { + source.insert((*field).to_string(), value.clone()); + } + } + (!source.is_empty()).then(|| Value::Object(source)) +} + +/// Signals that describe the prompt, which does not grow while the completion +/// streams. Repeating them on every chunk is common, so they are set once and +/// never concatenated — the completion-side signals are the ones that append. +const SET_ONCE_TOKEN_SIGNALS: &[&str] = &["prompt_token_ids"]; + +/// Merge any token-level RL signals found in `source` into `signals`. +/// +/// Completion-side signals accumulate across streamed chunks, because a +/// chat-completions server emits them one delta at a time; prompt-side signals +/// are set once. Every field is normalized to the flat array the OTS contract +/// requires: `logprobs` arrives from OpenAI-compatible servers as +/// `{"content": [{"token": …, "logprob": …}]}` and is flattened to the bare +/// logprob values. Shapes that cannot be normalized are ignored rather than +/// guessed at. +pub fn merge_token_signals(signals: &mut Option, source: &Value) { + for field in TOKEN_SIGNAL_FIELDS { + let Some(raw) = source.get(*field) else { + continue; + }; + let incoming = if *field == "logprobs" { + normalize_logprobs(raw) + } else { + // Token ids and mask bits are numbers. An endpoint streaming + // anything else under these names is not producing the signal the + // field names, and passing it through would put unbounded foreign + // text on a SessionEntry whose budget assumes numbers — and would + // be dropped by the emitter's own shape checks anyway. + raw.as_array() + .filter(|items| items.iter().all(Value::is_number)) + .cloned() + }; + let Some(incoming) = incoming.filter(|items| !items.is_empty()) else { + continue; + }; + let map = signals.get_or_insert_with(|| Value::Object(Map::new())); + let Some(map) = map.as_object_mut() else { + return; + }; + if SET_ONCE_TOKEN_SIGNALS.contains(field) { + map.entry((*field).to_string()) + .or_insert_with(|| Value::Array(incoming)); + continue; + } + map.entry((*field).to_string()) + .or_insert_with(|| Value::Array(Vec::new())); + if let Some(existing) = map.get_mut(*field).and_then(Value::as_array_mut) { + existing.extend(incoming); + } + } +} + +/// Flatten an OpenAI-compatible `logprobs` payload to bare logprob values. +fn normalize_logprobs(raw: &Value) -> Option> { + if let Some(items) = raw.as_array() { + if items.iter().all(Value::is_number) { + return Some(items.clone()); + } + return flatten_logprob_entries(items); + } + raw.get("content") + .and_then(Value::as_array) + .and_then(|items| flatten_logprob_entries(items)) +} + +/// Take the `logprob` of every entry, or nothing. +/// +/// The flattened array is positional: element *i* is the probability of +/// completion token *i*. Dropping one malformed entry would shift every later +/// probability onto the wrong token, which is worse than having no +/// probabilities at all — so a payload that cannot be flattened whole is +/// rejected whole. +fn flatten_logprob_entries(items: &[Value]) -> Option> { + items + .iter() + .map(|item| item.get("logprob").filter(|value| value.is_number()).cloned()) + .collect() } #[derive(Debug, Clone, PartialEq, Eq)] @@ -98,6 +216,8 @@ pub struct ChatCompletionStreamAccumulator { output_tokens: i64, saw_done: bool, semantic_deltas: Vec, + token_signals: Option, + response_id: String, } impl ChatCompletionStreamAccumulator { @@ -118,6 +238,13 @@ impl ChatCompletionStreamAccumulator { })?; let mut deltas = Vec::new(); + if self.response_id.is_empty() + && let Some(id) = event.get("id").and_then(Value::as_str) + && !id.is_empty() + { + self.response_id = id.to_string(); + } + if let Some(usage) = event.get("usage") { self.input_tokens = usage .get("prompt_tokens") @@ -131,6 +258,17 @@ impl ChatCompletionStreamAccumulator { .unwrap_or(self.output_tokens); } + // One event contributes each signal once — see `event_token_signals`. + if let Some(source) = event_token_signals( + event.get("usage"), + event + .get("choices") + .and_then(Value::as_array) + .and_then(|choices| choices.first()), + ) { + merge_token_signals(&mut self.token_signals, &source); + } + if let Some(choice) = event .get("choices") .and_then(Value::as_array) @@ -186,7 +324,7 @@ impl ChatCompletionStreamAccumulator { } Ok(ParsedChatCompletion { - content: chat_content_blocks(&self.text, &self.tool_calls), + content: chat_content_blocks(&self.text, &self.tool_calls, &self.response_id), stop_reason: if !self.tool_calls.is_empty() { "tool_use".to_string() } else { @@ -197,6 +335,7 @@ impl ChatCompletionStreamAccumulator { response_bytes, semantic_deltas: self.semantic_deltas, completed: true, + token_signals: self.token_signals, }) } } @@ -240,7 +379,26 @@ fn ingest_tool_call_deltas( } } -fn chat_content_blocks(text: &str, tool_calls: &BTreeMap) -> Vec { +/// Tool-call id for a provider that sent none. +/// +/// A bare positional id (`tool_1`) restarts at every response, so two turns that +/// each make one call end up sharing an id — and anything that indexes calls by +/// id across a session (the OTS emitter's decisions and tool spans, the wire +/// conversion back to chat format) then collapses them into one. The response id +/// scopes the fallback to the completion that produced it. +pub fn synthetic_tool_call_id(response_id: &str, scope: &str, position: usize) -> String { + if response_id.is_empty() { + format!("{scope}_{position}") + } else { + format!("{response_id}_{scope}_{position}") + } +} + +fn chat_content_blocks( + text: &str, + tool_calls: &BTreeMap, + response_id: &str, +) -> Vec { let mut content = Vec::::new(); if !text.is_empty() { content.push(json!({ @@ -257,7 +415,11 @@ fn chat_content_blocks(text: &str, tool_calls: &BTreeMap Vec< "content": system_prompt, })); } - for msg in messages { + for (message_index, msg) in messages.iter().enumerate() { let role = msg.get("role").and_then(Value::as_str).unwrap_or("user"); let content = msg.get("content").cloned().unwrap_or(json!("")); @@ -380,11 +542,20 @@ pub fn convert_messages_to_chat(system_prompt: &str, messages: &[Value]) -> Vec< } } "tool_use" => { + // Position within the message is not unique across a + // conversation; scope it so two id-less assistant + // turns cannot send the provider the same call id. let id = block .get("id") .and_then(Value::as_str) .map(str::to_string) - .unwrap_or_else(|| format!("tool_{}", idx + 1)); + .unwrap_or_else(|| { + synthetic_tool_call_id( + &format!("msg{message_index}"), + "tool", + idx + 1, + ) + }); let name = block .get("name") .and_then(Value::as_str) @@ -547,6 +718,195 @@ fn map_from_pairs(pairs: &[(String, String)]) -> Map { mod tests { use super::*; + #[test] + fn merge_token_signals_flattens_openai_logprobs() { + let mut signals = None; + merge_token_signals( + &mut signals, + &json!({ + "logprobs": { "content": [ + {"token": "he", "logprob": -0.25}, + {"token": "llo", "logprob": -1.5} + ]} + }), + ); + assert_eq!(signals.unwrap()["logprobs"], json!([-0.25, -1.5])); + } + + #[test] + fn merge_token_signals_accumulates_across_chunks() { + let mut signals = None; + merge_token_signals(&mut signals, &json!({ "logprobs": [-0.1] })); + merge_token_signals(&mut signals, &json!({ "logprobs": [-0.2] })); + merge_token_signals( + &mut signals, + &json!({ "completion_token_ids": [7, 8], "response_mask": [1, 1] }), + ); + let signals = signals.unwrap(); + assert_eq!(signals["logprobs"], json!([-0.1, -0.2])); + assert_eq!(signals["completion_token_ids"], json!([7, 8])); + assert_eq!(signals["response_mask"], json!([1, 1])); + } + + #[test] + fn merge_token_signals_sets_prompt_ids_once_instead_of_concatenating() { + // A server that repeats the prompt ids on every chunk must not end up + // with the prompt counted N times. + let mut signals = None; + for _ in 0..4 { + merge_token_signals(&mut signals, &json!({ "prompt_token_ids": [11, 12, 13] })); + } + assert_eq!(signals.unwrap()["prompt_token_ids"], json!([11, 12, 13])); + } + + /// Logprobs are positional: element *i* belongs to completion token *i*. + /// Skipping a malformed entry shifts every later probability onto the wrong + /// token and leaves the arrays at different lengths, so the payload is + /// dropped whole instead. + #[test] + fn merge_token_signals_drops_logprobs_that_cannot_be_flattened_whole() { + let mut signals = None; + merge_token_signals( + &mut signals, + &json!({ + "completion_token_ids": [7, 8, 9], + "response_mask": [1, 1, 1], + "logprobs": { "content": [ + {"token": "a", "logprob": -0.1}, + {"token": "b"}, + {"token": "c", "logprob": -0.3} + ]}, + }), + ); + let signals = signals.expect("token id signals still recorded"); + assert_eq!(signals["completion_token_ids"], json!([7, 8, 9])); + assert!( + signals.get("logprobs").is_none(), + "a partial logprob array must never be stored alongside full token ids" + ); + } + + /// Token ids and mask bits are numbers. An OpenAI-compatible endpoint is + /// configurable per agent, so what it streams under these names is not + /// trusted: text elements here would be unbounded foreign content written + /// onto a SessionEntry whose size budget assumes numbers, and the emitter's + /// own shape checks would drop them from the trajectory anyway. + #[test] + fn merge_token_signals_rejects_non_numeric_token_arrays() { + let mut signals = None; + merge_token_signals( + &mut signals, + &json!({ + "prompt_token_ids": ["hello", "world"], + "completion_token_ids": [7, 8], + "response_mask": [1, "0"], + }), + ); + let signals = signals.expect("the numeric signal is still recorded"); + assert_eq!(signals["completion_token_ids"], json!([7, 8])); + assert!( + signals.get("prompt_token_ids").is_none() + && signals.get("response_mask").is_none(), + "arrays carrying anything but numbers must be dropped whole: {signals}" + ); + } + + /// A server that echoes the same payload under both `usage` and + /// `choices[0]` of one event must not have it counted twice. With a single + /// signal present there is no second array to disagree on length, so the + /// doubling would reach an RL consumer as real token ids. + #[test] + fn one_event_contributes_each_token_signal_once() { + let mut acc = ChatCompletionStreamAccumulator::default(); + acc.ingest_data( + &json!({ + "usage": {"completion_token_ids": [7, 8]}, + "choices": [{"completion_token_ids": [7, 8], "delta": {"content": "hi"}}], + }) + .to_string(), + ) + .expect("event parses"); + + let signals = acc.token_signals.clone().expect("signals recorded"); + assert_eq!( + signals["completion_token_ids"], + json!([7, 8]), + "the repeated payload must be taken once, not concatenated" + ); + } + + /// Where both levels carry the same field the per-choice value wins, and a + /// field only `usage` carries is still picked up. + #[test] + fn choice_level_token_signals_win_over_usage_level() { + let source = event_token_signals( + Some(&json!({"completion_token_ids": [1], "prompt_token_ids": [5, 6]})), + Some(&json!({"completion_token_ids": [9]})), + ) + .expect("a source is produced"); + + assert_eq!(source["completion_token_ids"], json!([9]), "choice wins"); + assert_eq!( + source["prompt_token_ids"], + json!([5, 6]), + "a field only usage carries is still taken" + ); + assert!(event_token_signals(None, None).is_none()); + assert!(event_token_signals(Some(&json!({"prompt_tokens": 4})), None).is_none()); + } + + /// Across events the completion side still accumulates — that is what makes + /// a streamed completion whole. + #[test] + fn completion_signals_still_accumulate_across_events() { + let mut acc = ChatCompletionStreamAccumulator::default(); + for chunk in [json!({"choices": [{"completion_token_ids": [1]}]}), + json!({"choices": [{"completion_token_ids": [2]}]})] { + acc.ingest_data(&chunk.to_string()).expect("event parses"); + } + let signals = acc.token_signals.clone().expect("signals recorded"); + assert_eq!(signals["completion_token_ids"], json!([1, 2])); + } + + #[test] + fn merge_token_signals_stays_none_for_providers_that_send_nothing() { + let mut signals = None; + merge_token_signals( + &mut signals, + &json!({ "usage": {"prompt_tokens": 10}, "logprobs": null }), + ); + assert!(signals.is_none()); + } + + #[test] + fn chat_stream_accumulator_captures_streamed_logprobs() { + let mut accumulator = ChatCompletionStreamAccumulator::default(); + accumulator + .ingest_data( + r#"{"choices":[{"delta":{"content":"hi"},"logprobs":{"content":[{"token":"hi","logprob":-0.5}]}}]}"#, + ) + .expect("chunk parses"); + accumulator + .ingest_data( + r#"{"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1,"prompt_token_ids":[11,12,13]}}"#, + ) + .expect("final chunk parses"); + let parsed = accumulator.finalize(128).expect("stream finalizes"); + let signals = parsed.token_signals.expect("signals captured"); + assert_eq!(signals["logprobs"], json!([-0.5])); + assert_eq!(signals["prompt_token_ids"], json!([11, 12, 13])); + } + + #[test] + fn chat_stream_accumulator_leaves_signals_absent_without_them() { + let mut accumulator = ChatCompletionStreamAccumulator::default(); + accumulator + .ingest_data(r#"{"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]}"#) + .expect("chunk parses"); + let parsed = accumulator.finalize(64).expect("stream finalizes"); + assert!(parsed.token_signals.is_none()); + } + #[test] fn builds_chat_body_and_merges_safe_provider_options() { let body = build_chat_completion_body( @@ -610,6 +970,61 @@ mod tests { assert_eq!(parsed.content[1]["input"]["kind"], "Session"); } + /// A provider that omits tool-call ids used to get `tool_1` on every turn, + /// so a session's second call overwrote the first everywhere calls are keyed + /// by id — OTS decisions, tool spans, and the wire conversion back to chat. + #[test] + fn synthetic_tool_call_ids_are_scoped_to_their_response() { + let call_without_id = |completion: &str| { + let events = vec![ + format!( + r#"{{"id":"{completion}","choices":[{{"delta":{{"tool_calls":[{{"index":0,"function":{{"name":"temper_list","arguments":"{{}}"}}}}]}}}}]}}"# + ), + r#"{"choices":[{"finish_reason":"tool_calls","delta":{}}]}"#.to_string(), + "[DONE]".to_string(), + ]; + parse_chat_completion_stream_events(&events, 64).expect("stream parses").content[0] + ["id"] + .as_str() + .expect("tool_use id") + .to_string() + }; + + let first = call_without_id("chatcmpl-aaa"); + let second = call_without_id("chatcmpl-bbb"); + assert_eq!(first, "chatcmpl-aaa_tool_1"); + assert_ne!( + first, second, + "two responses that both omit call ids must not share one" + ); + } + + #[test] + fn synthetic_tool_call_ids_fall_back_when_the_response_has_no_id() { + assert_eq!(synthetic_tool_call_id("", "tool", 1), "tool_1"); + assert_eq!(synthetic_tool_call_id("gen-9", "or_tool", 2), "gen-9_or_tool_2"); + } + + /// Two assistant turns that both lost their call ids must not send the + /// provider one ambiguous id shared by both. + #[test] + fn converted_messages_scope_missing_tool_call_ids_per_message() { + let messages = vec![ + json!({"role":"assistant","content":[{"type":"tool_use","name":"a","input":{}}]}), + json!({"role":"assistant","content":[{"type":"tool_use","name":"b","input":{}}]}), + ]; + let converted = convert_messages_to_chat("", &messages); + let ids: Vec<&str> = converted + .iter() + .filter_map(|message| message.get("tool_calls")) + .filter_map(Value::as_array) + .flatten() + .filter_map(|call| call["id"].as_str()) + .collect(); + assert_eq!(ids.len(), 2); + assert_ne!(ids[0], ids[1]); + } + #[test] fn parses_non_stream_chat_completion_content() { let text = parse_chat_completion_response_text( diff --git a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml index 4dc870262..da5dae99e 100644 --- a/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml +++ b/os-apps/paw-agent/wasm/openai_codex_auth/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } openai-codex-wire = { path = "../openai-codex-wire" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml index dafb631fa..691e0c148 100644 --- a/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_approval_handler/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml index fe7ee5657..bc9225043 100644 --- a/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml +++ b/os-apps/paw-agent/wasm/plan_review_feedback_handler/Cargo.toml @@ -11,5 +11,5 @@ crate-type = ["cdylib"] [dependencies] serde_json = "1" session-tree-lib = { path = "../session-tree-lib" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml index 20f5ef4cc..818412e35 100644 --- a/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_auth_gate/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock index 672487d29..53ecfcbed 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.lock @@ -132,7 +132,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml index 668958881..3f429ffc3 100644 --- a/os-apps/paw-agent/wasm/provider_caller/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_caller/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_caller/src/lib.rs b/os-apps/paw-agent/wasm/provider_caller/src/lib.rs index 4af5321d0..a6c1b4f1d 100644 --- a/os-apps/paw-agent/wasm/provider_caller/src/lib.rs +++ b/os-apps/paw-agent/wasm/provider_caller/src/lib.rs @@ -12,7 +12,8 @@ use openai_chat_wire::{ ChatCompletionStreamAccumulator, ChatStreamDelta, ChatStreamParseFailure, - build_chat_completion_body, convert_messages_to_chat, parse_headers_json, + build_chat_completion_body, convert_messages_to_chat, event_token_signals, + merge_token_signals, parse_headers_json, synthetic_tool_call_id, }; #[cfg(test)] use openai_codex_wire::base64_url_no_pad; @@ -69,6 +70,8 @@ struct LlmResponse { cache_creation_input_tokens: i64, request_bytes: usize, response_bytes: usize, + /// Token-level RL signals the serving stack returned, when it returned any. + token_signals: Option, } fn normalize_provider(provider: &str) -> String { @@ -409,6 +412,8 @@ struct ParsedProviderStream { response_bytes: usize, semantic_deltas: Vec, completed: bool, + /// Token-level RL signals the serving stack streamed, when it streamed any. + token_signals: Option, } impl ParsedProviderStream { @@ -422,6 +427,7 @@ impl ParsedProviderStream { cache_creation_input_tokens: self.cache_creation_input_tokens, request_bytes, response_bytes: self.response_bytes, + token_signals: self.token_signals, } } } @@ -555,6 +561,7 @@ struct OpenAiStreamAccumulator { streamed_text: String, saw_completed: bool, semantic_deltas: Vec, + token_signals: Option, } impl OpenAiStreamAccumulator { @@ -618,6 +625,12 @@ impl OpenAiStreamAccumulator { "response.completed" => { self.saw_completed = true; if let Some(resp) = event.get("response") { + // One event contributes each signal once — the response is + // the content level, `response.usage` the accounting one. + // See `event_token_signals`. + if let Some(source) = event_token_signals(resp.get("usage"), Some(resp)) { + merge_token_signals(&mut self.token_signals, &source); + } if let Some(usage) = resp.get("usage") { self.usage = usage.clone(); } @@ -685,6 +698,7 @@ impl OpenAiStreamAccumulator { response_bytes, semantic_deltas: self.semantic_deltas, completed: true, + token_signals: self.token_signals, }) } } @@ -1001,6 +1015,8 @@ impl AnthropicStreamAccumulator { response_bytes, semantic_deltas: self.semantic_deltas, completed: true, + // The Anthropic Messages stream carries no token ids or logprobs. + token_signals: None, }) } } @@ -1017,6 +1033,18 @@ fn parse_anthropic_stream_events( acc.finalize(response_bytes) } +// The OpenRouter-specific path below is NOT the live one. `call_provider` +// dispatches "openrouter" to `call_openai_compatible_chat`, which uses the +// shared `openai_chat_wire` conversions and accumulator. Everything from here +// to `convert_tools_to_openrouter` is unreachable — the compiler says so on +// every build ("never used" / "never constructed"), and those warnings are left +// standing deliberately rather than silenced with `#[allow(dead_code)]`. +// +// It is kept, not deleted, because whether this copy has a future is +// nerdsane/temperpaw#459's call, not this branch's. It is still maintained to +// the same invariants — a reader who mistakes it for the live path must not +// find a stale rule in it. + #[derive(Default)] struct OpenRouterToolCallAccum { id: String, @@ -1033,6 +1061,8 @@ struct OpenRouterStreamAccumulator { output_tokens: i64, saw_done: bool, semantic_deltas: Vec, + token_signals: Option, + response_id: String, } impl OpenRouterStreamAccumulator { @@ -1050,6 +1080,13 @@ impl OpenRouterStreamAccumulator { })?; let mut deltas = Vec::new(); + if self.response_id.is_empty() + && let Some(id) = event.get("id").and_then(Value::as_str) + && !id.is_empty() + { + self.response_id = id.to_string(); + } + if let Some(usage) = event.get("usage") { self.input_tokens = usage .get("prompt_tokens") @@ -1063,6 +1100,17 @@ impl OpenRouterStreamAccumulator { .unwrap_or(self.output_tokens); } + // One event contributes each signal once — see `event_token_signals`. + if let Some(source) = event_token_signals( + event.get("usage"), + event + .get("choices") + .and_then(Value::as_array) + .and_then(|choices| choices.first()), + ) { + merge_token_signals(&mut self.token_signals, &source); + } + if let Some(choice) = event .get("choices") .and_then(Value::as_array) @@ -1145,7 +1193,11 @@ impl OpenRouterStreamAccumulator { }; content.push(json!({ "type": "tool_use", - "id": if tool_call.id.is_empty() { format!("or_tool_{}", idx + 1) } else { tool_call.id.clone() }, + "id": if tool_call.id.is_empty() { + synthetic_tool_call_id(&self.response_id, "or_tool", idx + 1) + } else { + tool_call.id.clone() + }, "name": if tool_call.name.is_empty() { "unknown_tool".to_string() } else { tool_call.name.clone() }, "input": input, })); @@ -1165,6 +1217,7 @@ impl OpenRouterStreamAccumulator { response_bytes, semantic_deltas: self.semantic_deltas, completed: true, + token_signals: self.token_signals, }) } } @@ -2438,9 +2491,12 @@ fn call_openai_compatible_chat( cache_creation_input_tokens: 0, request_bytes: body_str.len(), response_bytes: parsed.response_bytes, + token_signals: parsed.token_signals, }) } +/// Unreachable: `call_provider` sends "openrouter" to +/// `call_openai_compatible_chat`. See the note above `OpenRouterToolCallAccum`. fn call_openrouter( ctx: &Context, temper_api_url: &str, @@ -3488,6 +3544,7 @@ fn build_mock_step_response( cache_creation_input_tokens: 0, request_bytes: 0, response_bytes: serialized_content.len(), + token_signals: None, }); } @@ -3516,6 +3573,7 @@ fn mock_text_response(messages: &[Value], text: String) -> LlmResponse { cache_creation_input_tokens: 0, request_bytes: 0, response_bytes: text.len(), + token_signals: None, } } @@ -3592,9 +3650,12 @@ fn extract_memory_keys(text: &str) -> Vec { .collect() } +/// Unreachable, and kept in step with the live `convert_messages_to_chat` +/// anyway: the fallback tool-call id is scoped per message, because a reader +/// who mistakes this for the live path must not find the unscoped rule here. fn convert_messages_to_openrouter(messages: &[Value]) -> Vec { let mut out = Vec::::new(); - for msg in messages { + for (message_index, msg) in messages.iter().enumerate() { let role = msg.get("role").and_then(Value::as_str).unwrap_or("user"); let content = msg.get("content").cloned().unwrap_or(json!("")); @@ -3621,7 +3682,19 @@ fn convert_messages_to_openrouter(messages: &[Value]) -> Vec { .get("id") .and_then(Value::as_str) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("tool_{}", idx + 1)); + .unwrap_or_else(|| { + // Position within the message is not + // unique across a conversation; scope + // it so two id-less assistant turns + // cannot send the provider the same + // call id, which would collapse two + // decisions into one (ADR-0035 §14). + synthetic_tool_call_id( + &format!("msg{message_index}"), + "tool", + idx + 1, + ) + }); let name = block .get("name") .and_then(Value::as_str) @@ -4037,6 +4110,7 @@ pub fn run_provider_caller() -> Result<(), String> { cache_creation_input_tokens: response.cache_creation_input_tokens, request_bytes: response.request_bytes, response_bytes: response.response_bytes, + token_signals: response.token_signals, }; let artifact_json = serde_json::to_string(&artifact) .map_err(|e| format!("provider response artifact serialize: {e}"))?; @@ -4268,6 +4342,100 @@ mod tests { use super::*; + /// The dead OpenRouter conversion is kept in step with the live one. This + /// exact line regressed once already — a rebase restored a copy predating + /// the fix — so the invariant is pinned here rather than trusted to the + /// code being unreachable today. Position within a message is not unique + /// across a conversation: two id-less assistant turns would otherwise send + /// the provider the same call id, and the emitter would collapse two + /// decisions into one (ADR-0035 section 14). + #[test] + fn openrouter_conversion_scopes_missing_tool_call_ids_per_message() { + let messages = vec![ + json!({"role":"assistant","content":[{"type":"tool_use","name":"a","input":{}}]}), + json!({"role":"assistant","content":[{"type":"tool_use","name":"b","input":{}}]}), + ]; + let converted = convert_messages_to_openrouter(&messages); + let ids: Vec<&str> = converted + .iter() + .filter_map(|message| message.get("tool_calls")) + .filter_map(Value::as_array) + .flatten() + .filter_map(|call| call["id"].as_str()) + .collect(); + + assert_eq!(ids.len(), 2); + assert_ne!( + ids[0], ids[1], + "two id-less assistant turns must not share a synthetic call id" + ); + } + + /// A server that carries the same signals at both levels of one event must + /// not have them stored twice. Completion-side signals accumulate across + /// events, and with a single signal present there is no second array to + /// disagree on length — so the doubling would reach an RL consumer as real + /// token ids. Same class as the chat accumulator's; these two are the other + /// two wire shapes. + #[test] + fn openrouter_event_contributes_each_token_signal_once() { + let mut accumulator = OpenRouterStreamAccumulator::default(); + accumulator + .ingest_data( + &json!({ + "usage": {"completion_tokens": 2, "completion_token_ids": [7, 8]}, + "choices": [{"completion_token_ids": [7, 8], "delta": {"content": "hi"}}], + }) + .to_string(), + ) + .expect("event parses"); + + let signals = accumulator + .token_signals + .clone() + .expect("token signals recorded"); + assert_eq!( + signals["completion_token_ids"], + json!([7, 8]), + "the repeated payload must be taken once, not concatenated" + ); + } + + #[test] + fn openai_response_completed_contributes_each_token_signal_once() { + let mut accumulator = OpenAiStreamAccumulator::default(); + accumulator + .ingest_data( + &json!({ + "type": "response.completed", + "response": { + "completion_token_ids": [4, 5, 6], + "usage": { + "input_tokens": 3, + "output_tokens": 3, + "completion_token_ids": [4, 5, 6], + }, + }, + }) + .to_string(), + ) + .expect("event parses"); + + let signals = accumulator + .token_signals + .clone() + .expect("token signals recorded"); + assert_eq!( + signals["completion_token_ids"], + json!([4, 5, 6]), + "response and response.usage are one event, not two measurements" + ); + assert_eq!( + accumulator.usage["output_tokens"], 3, + "usage accounting is still captured" + ); + } + #[test] fn provider_progress_wrapper_emits_start_and_end_on_success() { let mut events = Vec::new(); diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock index 568007a07..80ea8daeb 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.lock @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml index 59056ef0b..95c382ed9 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml +++ b/os-apps/paw-agent/wasm/provider_response_applier/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } session-turn-artifacts = { path = "../session-turn-artifacts" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs b/os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs index 965285923..059452051 100644 --- a/os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs +++ b/os-apps/paw-agent/wasm/provider_response_applier/src/lib.rs @@ -17,10 +17,11 @@ use session_turn_artifacts::{ }; use temper_wasm_sdk::prelude::*; use wasm_helpers::{ - append_session_entry_inline, create_content_file, is_session_entries_ref, - materialize_initial_session_entries_with_assistant, read_content_file, + MAX_ENTRY_EXTRA_BYTES, append_session_entry_inline, create_content_file, + is_session_entries_ref, materialize_initial_session_entries_with_assistant, read_content_file, read_session_from_temperfs, resolve_temper_api_url, runtime_headers, - session_id_from_entries_ref, write_session_to_temperfs, write_temperfs_value_with_retry, + session_id_from_entries_ref, stored_json_len as escaped_json_len, write_session_to_temperfs, + write_temperfs_value_with_retry, }; const SESSION_ENTRY_FILE_THRESHOLD_BYTES: usize = 4096; @@ -161,8 +162,7 @@ pub fn run_provider_response_applier() -> Result<(), String> { &temper_api_url, tenant, &fields, - &response.content, - response.output_tokens as usize, + &response, ); emit_phase_step_duration( &ctx, @@ -208,8 +208,7 @@ pub fn run_provider_response_applier() -> Result<(), String> { &temper_api_url, tenant, &fields, - &response.content, - response.output_tokens as usize, + &response, ); emit_phase_step_duration( &ctx, @@ -378,13 +377,17 @@ fn append_assistant_response_to_session_tree( temper_api_url: &str, tenant: &str, fields: &Value, - content: &Value, - output_tokens: usize, + response: &ProviderResponseArtifact, ) -> Result, String> { if !prepared.use_session_tree { return Ok(None); } + let content = &response.content; + let output_tokens = response.output_tokens.max(0) as usize; + let extra = assistant_turn_extra(response, Context::get_time_millis()); + let extra = Some(&extra); + if is_session_entries_ref(&prepared.session_file_id) { if !session_entries_materialized(fields) { let session_id = session_id_from_entries_ref(&prepared.session_file_id) @@ -399,6 +402,7 @@ fn append_assistant_response_to_session_tree( &user_message, content, output_tokens, + extra, )?; ctx.log( "info", @@ -420,6 +424,7 @@ fn append_assistant_response_to_session_tree( "assistant", content, output_tokens, + extra, )?; return Ok(Some(created.entry_id)); } @@ -452,21 +457,27 @@ fn append_assistant_response_to_session_tree( &content_file_id, None, output_tokens, + extra, ); (leaf, true) } Err(_) => { - let (leaf, _) = tree.append_assistant_message( + let (leaf, _) = tree.append_assistant_message_with_extra( &prepared.session_leaf_id, content, output_tokens, + extra, ); (leaf, false) } } } else { - let (leaf, _) = - tree.append_assistant_message(&prepared.session_leaf_id, content, output_tokens); + let (leaf, _) = tree.append_assistant_message_with_extra( + &prepared.session_leaf_id, + content, + output_tokens, + extra, + ); (leaf, false) }; @@ -491,6 +502,87 @@ fn append_assistant_response_to_session_tree( Ok(Some(new_leaf)) } +/// Ceiling on a single token-level signal array stored on a SessionEntry. +/// +/// These arrays scale with completion length, and the entry's ExtraJson has its +/// own overflow ceiling; a long completion's logprobs must not be what pushes a +/// turn over it. +const MAX_TOKEN_SIGNAL_BYTES: usize = 32_768; + +// The entry's `extra_json` ceiling (`MAX_ENTRY_EXTRA_BYTES`) is spent here as +// policy: bounding each signal on its own does not bound their sum — four +// signals just under the per-signal ceiling each pass and cross the entry +// ceiling together — and past it the kernel replaces or externalizes the +// *entire* field, taking the per-turn facts the OTS emitter needs with it. +// Choosing which signal to sacrifice, and naming it, belongs to this writer; +// `wasm_helpers` enforces the same ceiling at the write boundary for every +// writer, including ones that never come through here. + +/// Headroom withheld from `MAX_ENTRY_EXTRA_BYTES`. +/// +/// `create_session_entry` stamps `recorded_at` onto the object after this +/// function has returned, and the `_dropped_bytes` markers written +/// below are themselves not charged against the budget. Escaping is not part of +/// the headroom — `escaped_json_len` accounts for it exactly. +const ENTRY_EXTRA_HEADROOM_BYTES: usize = 4_096; + +/// Per-turn facts recorded on the assistant SessionEntry. +/// +/// The OTS emitter reads these back to date each turn, report its prompt and +/// completion token counts, and carry token-level RL signals when the serving +/// stack produced them. Everything here is already in hand — recording it costs +/// no extra provider or storage round trip. `now_ms` is passed in rather than +/// read here so the mapping stays testable off-host. +fn assistant_turn_extra(response: &ProviderResponseArtifact, now_ms: i64) -> Value { + let mut extra = json!({ + "ts_ms": now_ms, + "provider": response.provider, + "model": response.model, + "stop_reason": response.stop_reason, + "input_tokens": response.input_tokens.max(0), + "output_tokens": response.output_tokens.max(0), + }); + if response.cache_read_input_tokens > 0 { + extra["cache_read_input_tokens"] = json!(response.cache_read_input_tokens); + } + if response.cache_creation_input_tokens > 0 { + extra["cache_creation_input_tokens"] = json!(response.cache_creation_input_tokens); + } + if let Some(Value::Object(signals)) = response.token_signals.clone() { + // Signals are added against a running total, so the entry keeps as many + // as fit and the ones that do not fit are named. The per-turn facts + // above are never at risk: they are already in the object, and nothing + // below can push the value past the ceiling. + let mut remaining = MAX_ENTRY_EXTRA_BYTES + .saturating_sub(ENTRY_EXTRA_HEADROOM_BYTES) + .saturating_sub(escaped_json_len(&extra)); + for (key, value) in signals { + let size = serde_json::to_string(&value).map(|json| json.len()).unwrap_or(0); + // The key, quotes, colon and separator ride along with the value, + // and the kernel measures the field after JSON-escaping it. + let cost = escaped_json_len(&value) + key.len() + 4; + let dropped = if size > MAX_TOKEN_SIGNAL_BYTES || cost > remaining { + true + } else { + remaining -= cost; + false + }; + let Some(target) = extra.as_object_mut() else { + break; + }; + if dropped { + // Record that it existed and how big it was; a dropped signal + // that leaves a trace is debuggable, a silent one is not. + target.insert(format!("{key}_dropped_bytes"), json!(size)); + continue; + } + target.insert(key, value); + } + } + extra +} + + fn extract_tool_calls(content: &Value) -> Vec { content .as_array() @@ -898,6 +990,237 @@ fn note_phase_budget_overrun_after_committed_step( mod tests { use super::*; + fn artifact_with_signals(token_signals: Option) -> ProviderResponseArtifact { + ProviderResponseArtifact { + version: 1, + provider: "anthropic".to_string(), + model: "claude-sonnet-4-6".to_string(), + content: json!([{"type": "text", "text": "done"}]), + stop_reason: "end_turn".to_string(), + input_tokens: 120, + output_tokens: 34, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + request_bytes: 256, + response_bytes: 512, + token_signals, + } + } + + #[test] + fn assistant_turn_extra_records_the_facts_the_emitter_needs() { + let extra = assistant_turn_extra(&artifact_with_signals(None), 1_767_225_600_000); + assert_eq!(extra["provider"], "anthropic"); + assert_eq!(extra["model"], "claude-sonnet-4-6"); + assert_eq!(extra["stop_reason"], "end_turn"); + assert_eq!(extra["input_tokens"], 120); + assert_eq!(extra["output_tokens"], 34); + assert_eq!(extra["ts_ms"], 1_767_225_600_000_i64, "turns must be datable"); + assert!(extra.get("logprobs").is_none()); + } + + #[test] + fn assistant_turn_extra_carries_token_signals_when_present() { + let extra = assistant_turn_extra( + &artifact_with_signals(Some(json!({ + "logprobs": [-0.5, -1.25], + "completion_token_ids": [7, 8], + }))), + 1_767_225_600_000, + ); + assert_eq!(extra["logprobs"], json!([-0.5, -1.25])); + assert_eq!(extra["completion_token_ids"], json!([7, 8])); + } + + /// The per-signal ceiling is applied to each signal on its own, and the ids + /// are several times the size of the mask — so there is a completion length + /// at which the ids are refused and the mask survives. That leaves an + /// anchorless set, which the kernel rejects outright: the emitter has to + /// drop it rather than pass it on, and this test pins that the writer really + /// does produce that shape, so the emitter's guard is not guarding a + /// hypothetical. + #[test] + fn assistant_turn_extra_can_keep_a_mask_after_refusing_its_token_ids() { + // Roughly an 8,000-token completion: five-digit ids plus separators run + // past the ceiling, a binary mask does not. + let ids: Vec = (0..8_000).map(|i| json!(10_000 + (i % 50_000))).collect(); + let mask: Vec = (0..8_000).map(|_| json!(1)).collect(); + let ids_bytes = serde_json::to_string(&ids).unwrap().len(); + let mask_bytes = serde_json::to_string(&mask).unwrap().len(); + assert!( + ids_bytes > MAX_TOKEN_SIGNAL_BYTES && mask_bytes <= MAX_TOKEN_SIGNAL_BYTES, + "fixture must straddle the ceiling: ids={ids_bytes} mask={mask_bytes}" + ); + + let extra = assistant_turn_extra( + &artifact_with_signals(Some(json!({ + "completion_token_ids": ids, + "response_mask": mask, + }))), + 1_767_225_600_000, + ); + + assert!( + extra.get("completion_token_ids").is_none(), + "the ids are over the per-signal ceiling and are refused" + ); + assert!( + extra["completion_token_ids_dropped_bytes"].as_u64().unwrap() > 0, + "and the refusal is recorded so the emitter can see it" + ); + assert!( + extra["response_mask"].is_array(), + "the mask fits, so the writer keeps it — this is the anchorless set \ + the emitter must not pass to the kernel" + ); + } + + #[test] + fn assistant_turn_extra_drops_oversized_token_signals() { + let huge: Vec = (0..MAX_TOKEN_SIGNAL_BYTES).map(|i| json!(i % 10)).collect(); + let extra = assistant_turn_extra( + &artifact_with_signals(Some(json!({ + "logprobs": huge, + "completion_token_ids": [1, 2, 3], + }))), + 1_767_225_600_000, + ); + assert!( + extra.get("logprobs").is_none(), + "an oversized signal must not be written to the entity" + ); + assert!( + extra["logprobs_dropped_bytes"].as_u64().unwrap() > MAX_TOKEN_SIGNAL_BYTES as u64, + "the drop must leave a trace" + ); + assert_eq!( + extra["completion_token_ids"], + json!([1, 2, 3]), + "a signal that fits still gets recorded" + ); + } + + /// Four signals that each clear the per-signal ceiling still cross the + /// entry's own ceiling together. Past it the kernel replaces or + /// externalizes the whole `extra_json` value, so the per-turn facts go with + /// them — the turn loses its timestamp, provider, model and token counts + /// because of signals nothing was even asking for. + #[test] + fn assistant_turn_extra_bounds_signals_against_the_entry_ceiling() { + // Single-digit elements serialize to two bytes each, so this lands just + // under the per-signal ceiling: every one of these passes the individual + // check, and four of them do not fit the entry together. + let near_ceiling: Vec = (0..MAX_TOKEN_SIGNAL_BYTES / 2 - 8) + .map(|i| json!(i % 10)) + .collect(); + assert!( + serde_json::to_string(&near_ceiling).unwrap().len() <= MAX_TOKEN_SIGNAL_BYTES, + "the fixture has to clear the per-signal ceiling for the test to mean anything" + ); + let extra = assistant_turn_extra( + &artifact_with_signals(Some(json!({ + "prompt_token_ids": near_ceiling, + "completion_token_ids": near_ceiling, + "response_mask": near_ceiling, + "logprobs": near_ceiling, + }))), + 1_767_225_600_000, + ); + + // Measured the way the kernel measures it: the field is a JSON string, + // so the ceiling applies to the escaped encoding. + let size = escaped_json_len(&extra); + assert!( + size <= MAX_ENTRY_EXTRA_BYTES - ENTRY_EXTRA_HEADROOM_BYTES, + "extra_json must stay under the entry ceiling, got {size} bytes" + ); + assert_eq!( + extra["ts_ms"], 1_767_225_600_000_i64, + "the per-turn facts must survive whatever the signals do" + ); + assert_eq!(extra["provider"], "anthropic"); + assert_eq!(extra["input_tokens"], 120); + + let dropped: Vec<&String> = extra + .as_object() + .unwrap() + .keys() + .filter(|key| key.ends_with("_dropped_bytes")) + .collect(); + assert!( + !dropped.is_empty(), + "signals that did not fit must name themselves: {extra}" + ); + for key in dropped { + let signal = key.trim_end_matches("_dropped_bytes"); + assert!( + extra.get(signal).is_none(), + "{signal} must not be both written and reported dropped" + ); + } + } + + /// The kernel measures `extra_json` after encoding it as a JSON string, so + /// a quote-dense value costs more stored bytes than it serializes to. A + /// budget that counts the unescaped length would let such a value cross the + /// ceiling and take the whole field — per-turn facts included — with it. + #[test] + fn entry_extra_budget_counts_escaped_bytes() { + // Ground truth: what the kernel stores is the extras JSON encoded again + // as a JSON string, which is what its overflow ceiling measures. + for value in [json!("\"\"\"\"\"\"\"\""), json!("\n"), json!({"a": [1, 2]})] { + let inner = serde_json::to_string(&value).unwrap(); + let stored = serde_json::to_string(&Value::String(inner)).unwrap(); + assert_eq!( + escaped_json_len(&value), + stored.len(), + "escaped size must match the encoding the kernel measures for {value}" + ); + } + + // A signal of quote-heavy strings: rejected at capture, and bounded + // here as a second line of defence. + let dense: Vec = (0..MAX_TOKEN_SIGNAL_BYTES / 8) + .map(|_| json!("\"\"\"")) + .collect(); + let extra = assistant_turn_extra( + &artifact_with_signals(Some(json!({ + "prompt_token_ids": dense.clone(), + "completion_token_ids": dense.clone(), + "response_mask": dense.clone(), + "logprobs": dense, + }))), + 1_767_225_600_000, + ); + let size = escaped_json_len(&extra); + assert!( + size <= MAX_ENTRY_EXTRA_BYTES - ENTRY_EXTRA_HEADROOM_BYTES, + "escaped extra_json must stay under the entry ceiling, got {size} bytes" + ); + assert_eq!(extra["ts_ms"], 1_767_225_600_000_i64); + } + + /// The ceiling is the spec's, not a number of this module's own choosing. + /// (It equals the kernel's default field ceiling too, so the bound holds + /// whichever of the two applies to this write path.) + #[test] + fn entry_extra_ceiling_matches_the_session_entry_spec() { + let spec = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../specs/session_entry.ioa.toml" + )) + .expect("session_entry.ioa.toml should exist"); + let extra_json_block = spec + .split("[[state]]") + .find(|block| block.contains("name = \"extra_json\"")) + .expect("session_entry.ioa.toml should declare extra_json"); + assert!( + extra_json_block + .contains(&format!("overflow_inline_max_bytes = \"{MAX_ENTRY_EXTRA_BYTES}\"")), + "MAX_ENTRY_EXTRA_BYTES must track the extra_json overflow ceiling: {extra_json_block}" + ); + } + #[test] fn extracts_tool_calls_only() { let tool_calls = extract_tool_calls(&json!([ @@ -1131,6 +1454,7 @@ mod tests { cache_creation_input_tokens: 0, request_bytes: 256, response_bytes: 512, + token_signals: None, }; assert!(legacy_updated_conversation_payload(&prepared, &artifact).is_none()); @@ -1168,6 +1492,7 @@ mod tests { cache_creation_input_tokens: 0, request_bytes: 256, response_bytes: 512, + token_signals: None, }; let payload = legacy_updated_conversation_payload(&prepared, &artifact) diff --git a/os-apps/paw-agent/wasm/request_approval/Cargo.toml b/os-apps/paw-agent/wasm/request_approval/Cargo.toml index f3fedb843..55308a30f 100644 --- a/os-apps/paw-agent/wasm/request_approval/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_approval/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml index f6a3d6a56..27cb55cc8 100644 --- a/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml +++ b/os-apps/paw-agent/wasm/request_plan_review/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml index 029fcd983..7fa2fda5f 100644 --- a/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/sandbox_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session-tree-lib/src/lib.rs b/os-apps/paw-agent/wasm/session-tree-lib/src/lib.rs index 92e3827cd..418ba4c92 100644 --- a/os-apps/paw-agent/wasm/session-tree-lib/src/lib.rs +++ b/os-apps/paw-agent/wasm/session-tree-lib/src/lib.rs @@ -452,6 +452,18 @@ impl SessionTree { parent_id: &str, content: &Value, tokens: usize, + ) -> (String, String) { + self.append_assistant_message_with_extra(parent_id, content, tokens, None) + } + + /// Append an assistant message carrying per-turn extras (provider, model, + /// stop reason, usage, token-level RL signals) the OTS emitter reads later. + pub fn append_assistant_message_with_extra( + &mut self, + parent_id: &str, + content: &Value, + tokens: usize, + extra_fields: Option<&Value>, ) -> (String, String) { let id = format!("a-{}", self.order.len()); let line = self.append_entry( @@ -461,7 +473,7 @@ impl SessionTree { Some("assistant"), Some(content), tokens, - None, + extra_fields, ); (id, line) } @@ -473,6 +485,7 @@ impl SessionTree { content_file_id: &str, content_file_version_id: Option<&str>, tokens: usize, + extra_fields: Option<&Value>, ) -> (String, String) { let id = format!("a-{}", self.order.len()); let line = self.append_entry_with_file( @@ -483,7 +496,7 @@ impl SessionTree { content_file_id, content_file_version_id, tokens, - None, + extra_fields, ); (id, line) } diff --git a/os-apps/paw-agent/wasm/session-turn-artifacts/src/lib.rs b/os-apps/paw-agent/wasm/session-turn-artifacts/src/lib.rs index 1e393d98b..01f7619dd 100644 --- a/os-apps/paw-agent/wasm/session-turn-artifacts/src/lib.rs +++ b/os-apps/paw-agent/wasm/session-turn-artifacts/src/lib.rs @@ -39,6 +39,44 @@ pub struct ProviderResponseArtifact { pub cache_creation_input_tokens: i64, pub request_bytes: usize, pub response_bytes: usize, + /// Token-level RL signals the serving stack returned with this completion + /// (`prompt_token_ids`, `completion_token_ids`, `response_mask`, + /// `logprobs`). Absent for providers that do not expose them — the agent + /// never makes an extra round trip to obtain them. See ADR-0035 section 10. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_signals: Option, +} + +/// Field names of the token-level RL signals carried between the provider +/// response and the OTS turn. The kernel's `OTSTurn` uses the same names. +pub const TOKEN_SIGNAL_FIELDS: &[&str] = &[ + "prompt_token_ids", + "completion_token_ids", + "response_mask", + "logprobs", +]; + +/// Pick the token-level RL signals out of a raw provider payload. +/// +/// Returns `None` when the payload carries none of them, which is the normal +/// case for the Anthropic-shaped providers this stack talks to today. +pub fn extract_token_signals(raw: &Value) -> Option { + let mut signals = serde_json::Map::new(); + for field in TOKEN_SIGNAL_FIELDS { + let value = raw + .get(*field) + .or_else(|| raw.get("usage").and_then(|usage| usage.get(*field))) + .or_else(|| raw.get("choices").and_then(|choices| { + choices + .as_array() + .and_then(|items| items.first()) + .and_then(|choice| choice.get(*field)) + })); + if let Some(value) = value.filter(|value| value.is_array()) { + signals.insert((*field).to_string(), value.clone()); + } + } + (!signals.is_empty()).then(|| Value::Object(signals)) } pub fn parse_prepared_context_artifact(raw: &str) -> Result { @@ -759,6 +797,7 @@ mod tests { cache_creation_input_tokens: 0, request_bytes: 256, response_bytes: 512, + token_signals: None, }; let params = @@ -898,6 +937,7 @@ mod tests { cache_creation_input_tokens: 0, request_bytes: 256, response_bytes: 512, + token_signals: None, }; let params = build_provider_response_applier_base_params(&prepared, &artifact); diff --git a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml index 036182b41..b609db675 100644 --- a/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_link_monitor/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml index 8b64818a0..954de8730 100644 --- a/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml +++ b/os-apps/paw-agent/wasm/session_recoverer/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml index 2aab0ea36..e59b672c4 100644 --- a/os-apps/paw-agent/wasm/steering_checker/Cargo.toml +++ b/os-apps/paw-agent/wasm/steering_checker/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../session-tree-lib" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml index 0a9b86d29..8d2497246 100644 --- a/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml +++ b/os-apps/paw-agent/wasm/wasm-helpers/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["rlib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs b/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs index 25b65f89e..9cb008896 100644 --- a/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs +++ b/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs @@ -48,6 +48,7 @@ pub struct CreatedSessionEntry { pub entry_id: String, } +#[derive(Clone, Copy)] struct SessionEntryCreateSpec<'a> { session_id: &'a str, entry_id: &'a str, @@ -113,16 +114,31 @@ fn read_temperfs_value_with_retry( headers: &[(String, String)], label: &str, ) -> Result { + read_temperfs_value_or_absent(ctx, url, headers, label).map(Option::unwrap_or_default) +} + +/// Same read, with a missing file reported as `None` instead of an empty body. +/// +/// A writer starting a new tree treats both the same way, which is why the +/// wrapper above collapses them. A reader deciding whether a record is complete +/// cannot: an absent file is missing history, an empty one is a session that has +/// not written history yet. +fn read_temperfs_value_or_absent( + ctx: &Context, + url: &str, + headers: &[(String, String)], + label: &str, +) -> Result, String> { let mut last_status = 0; let mut last_body = String::new(); for attempt in 0..TEMPERFS_READ_ATTEMPTS { let resp = ctx.http_call("GET", url, headers, "")?; if resp.status == 200 { - return Ok(resp.body); + return Ok(Some(resp.body)); } if resp.status == 404 { - return Ok(String::new()); + return Ok(None); } last_status = resp.status; @@ -146,7 +162,7 @@ fn read_temperfs_value_with_retry( Err(format!( "{label} (HTTP {}): {}", last_status, - &last_body[..last_body.len().min(200)] + error_excerpt(&last_body, 200) )) } @@ -193,7 +209,7 @@ pub fn write_temperfs_value_with_retry( Err(format!( "{label} (HTTP {}): {}", last_status, - &last_body[..last_body.len().min(200)] + error_excerpt(&last_body, 200) )) } @@ -214,7 +230,62 @@ pub fn resolve_temper_api_url(ctx: &Context, fields: &Value) -> String { .unwrap_or_else(|| "http://127.0.0.1:3000".to_string()) } +/// Why a session transcript read produced no entries — or that it produced some. +/// +/// A transcript has several ways of being empty and they are not the same fact. +/// A consumer that stores the result as a record of what an agent did has to be +/// able to tell "nothing has been written yet" from "what was written is gone". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscriptPresence { + /// The read returned at least one transcript line. + Present, + /// The transcript is SessionEntry rows and none are materialized yet — a + /// first-turn session that has not written one. + PendingFirstTurn, + /// The transcript is SessionEntry rows, materialization is on, and the + /// query returned nothing. + NoEntries, + /// The legacy TemperFS file the session points at does not exist. + MissingFile, + /// The legacy TemperFS file exists and holds nothing. + EmptyFile, + /// The session declares no transcript reference at all. + Undeclared, +} + +impl TranscriptPresence { + /// True only when the read produced transcript content. + pub fn is_present(self) -> bool { + matches!(self, TranscriptPresence::Present) + } + + /// Stable wire name, used in logs and in the emitted trajectory. + pub fn as_str(self) -> &'static str { + match self { + TranscriptPresence::Present => "present", + TranscriptPresence::PendingFirstTurn => "pending_first_turn", + TranscriptPresence::NoEntries => "no_entries", + TranscriptPresence::MissingFile => "missing_file", + TranscriptPresence::EmptyFile => "empty_file", + TranscriptPresence::Undeclared => "undeclared", + } + } +} + +/// A session transcript plus whether it is actually there. +#[derive(Debug, Clone)] +pub struct TranscriptRead { + /// Transcript JSONL, empty for every non-`Present` presence. + pub jsonl: String, + /// Whether the transcript was found, and if not, why not. + pub presence: TranscriptPresence, +} + /// Read session JSONL from TemperFS by file ID. +/// +/// Every "no transcript" case collapses to an empty string, which is what a +/// writer resuming a tree wants. Readers that have to distinguish absence from +/// emptiness use [`read_session_transcript`]. pub fn read_session_from_temperfs( ctx: &Context, temper_api_url: &str, @@ -222,6 +293,18 @@ pub fn read_session_from_temperfs( fields: &Value, file_id: &str, ) -> Result { + read_session_transcript(ctx, temper_api_url, tenant, fields, file_id) + .map(|read| read.jsonl) +} + +/// Read session JSONL and report whether the transcript exists. +pub fn read_session_transcript( + ctx: &Context, + temper_api_url: &str, + tenant: &str, + fields: &Value, + file_id: &str, +) -> Result { if let Some(session_id) = session_id_from_entries_ref(file_id) { if !session_entries_materialized(fields) { ctx.log( @@ -230,14 +313,40 @@ pub fn read_session_from_temperfs( "read_session_from_temperfs: virtual first-turn SessionEntries ref for {session_id}; materialization is false" ), ); - return Ok(String::new()); + return Ok(TranscriptRead { + jsonl: String::new(), + presence: TranscriptPresence::PendingFirstTurn, + }); } - return read_session_from_entries(ctx, temper_api_url, tenant, fields, session_id); + let jsonl = read_session_from_entries(ctx, temper_api_url, tenant, fields, session_id)?; + return Ok(TranscriptRead { + presence: transcript_presence(&jsonl, TranscriptPresence::NoEntries), + jsonl, + }); } let url = format!("{temper_api_url}/tdata/Files('{file_id}')/$value"); let headers = runtime_headers(ctx, tenant, fields, None, None); - read_temperfs_value_with_retry(ctx, &url, &headers, "TemperFS session read failed") + match read_temperfs_value_or_absent(ctx, &url, &headers, "TemperFS session read failed")? { + Some(jsonl) => Ok(TranscriptRead { + presence: transcript_presence(&jsonl, TranscriptPresence::EmptyFile), + jsonl, + }), + None => Ok(TranscriptRead { + jsonl: String::new(), + presence: TranscriptPresence::MissingFile, + }), + } +} + +/// A transcript of nothing but whitespace carries no entries, so it counts as +/// the caller's "empty" case rather than as content. +fn transcript_presence(jsonl: &str, when_empty: TranscriptPresence) -> TranscriptPresence { + if jsonl.lines().any(|line| !line.trim().is_empty()) { + TranscriptPresence::Present + } else { + when_empty + } } /// Write session JSONL to TemperFS by file ID. @@ -282,6 +391,11 @@ pub fn create_session_entry( extra_json: Option<&Value>, tokens: usize, ) -> Result { + // Every entry carries its own wall-clock stamp. The entity event log is a + // hot tail (older events vanish at snapshot boundaries), so it cannot be + // trusted to date turns in a long session; the entry can. + let extra_json = stamp_recorded_at(extra_json, Context::get_time_millis()); + let extra_json = Some(&extra_json); let spec = SessionEntryCreateSpec { session_id, entry_id, @@ -309,7 +423,7 @@ pub fn create_session_entry( return Err(format!( "SessionEntry creation failed (HTTP {}): {}", resp.status, - &resp.body[..resp.body.len().min(300)] + error_excerpt(&resp.body, 300) )); } let created = parse_created_session_entry_ack(&resp.body, session_id, entry_id)?; @@ -397,6 +511,7 @@ pub fn materialize_initial_session_entries_with_assistant( user_message: &str, assistant_content: &Value, assistant_tokens: usize, + assistant_extra_json: Option<&Value>, ) -> Result { let user_entry_id = format!("u-{session_id}-0"); let (assistant_entry_id, assistant_sequence) = next_session_entry_id("a", &user_entry_id); @@ -425,7 +540,7 @@ pub fn materialize_initial_session_entries_with_assistant( content: Some(assistant_content), content_file_id: None, content_file_version_id: None, - extra_json: None, + extra_json: assistant_extra_json, tokens: assistant_tokens, }, ]; @@ -473,10 +588,21 @@ fn create_session_entry_batch( Some("application/json"), ); let create_url = format!("{temper_api_url}/tdata/SessionEntries"); + // Same wall-clock stamp as the single-entry path — see create_session_entry. + let now_ms = Context::get_time_millis(); + let stamped: Vec = specs + .iter() + .map(|spec| stamp_recorded_at(spec.extra_json, now_ms)) + .collect(); let create_requests = specs .iter() - .map(|spec| { - let body = session_entry_create_body(spec)?; + .zip(stamped.iter()) + .map(|(spec, extra_json)| { + let spec = SessionEntryCreateSpec { + extra_json: Some(extra_json), + ..*spec + }; + let body = session_entry_create_body(&spec)?; Ok(HttpRequest { method: "POST".to_string(), url: create_url.clone(), @@ -501,7 +627,7 @@ fn create_session_entry_batch( "{label} {} creation failed (HTTP {}): {}", spec.entry_id, resp.status, - &resp.body[..resp.body.len().min(300)] + error_excerpt(&resp.body, 300) )); } created.push(parse_created_session_entry_ack( @@ -617,6 +743,224 @@ fn verify_session_entries( )) } +/// Add `ts_ms` to a SessionEntry's extra JSON without disturbing what the +/// caller already put there. An explicit `ts_ms` from the caller wins. +fn stamp_recorded_at(extra_json: Option<&Value>, now_ms: i64) -> Value { + let mut extra = match extra_json { + Some(Value::Object(map)) => Value::Object(map.clone()), + Some(other) if !other.is_null() => json!({ "extra": other.clone() }), + _ => json!({}), + }; + if let Some(object) = extra.as_object_mut() + && !object.contains_key("ts_ms") + { + object.insert("ts_ms".to_string(), json!(now_ms)); + } + extra +} + +/// Ceiling on a SessionEntry's stored `extra_json`. +/// +/// The same 131072 twice over: it is what `session_entry.ioa.toml` declares as +/// `overflow_inline_max_bytes` on the `extra_json` state variable, and it is +/// also the kernel's own `DEFAULT_FIELD_INLINE_MAX` when no per-field override +/// applies to a write path. The bound therefore holds whichever of the two the +/// kernel ends up measuring against. +pub const MAX_ENTRY_EXTRA_BYTES: usize = 131_072; + +/// Per-turn facts the OTS emitter reads back off an entry. They are small, and +/// losing them costs a turn its date, its model and its token counts, so they +/// are the last thing dropped rather than the first. +const ENTRY_EXTRA_ESSENTIALS: &[&str] = &[ + "ts_ms", + "provider", + "model", + "stop_reason", + "input_tokens", + "output_tokens", +]; + +/// The markers a token-signal refusal leaves behind. Bounded to one per signal, +/// and the emitter reads them to tell "the serving stack sent none" from "they +/// were sent and refused" — so the hard floor keeps them. +const TOKEN_SIGNAL_DROP_MARKERS: &[&str] = &[ + "prompt_token_ids_dropped_bytes", + "completion_token_ids_dropped_bytes", + "response_mask_dropped_bytes", + "logprobs_dropped_bytes", +]; + +/// Size of `value` as the kernel measures the stored field. +/// +/// String-typed state variables hold JSON as text, so what the overflow ceiling +/// sees is this JSON encoded *again* as a JSON string: every quote and +/// backslash gains an escape byte, plus the two enclosing quotes. (`serde_json` +/// output carries no raw control characters, so those are not a case.) +pub fn stored_json_len(value: &Value) -> usize { + let Ok(json) = serde_json::to_string(value) else { + return usize::MAX; + }; + json.len() + + json + .bytes() + .filter(|byte| matches!(byte, b'"' | b'\\')) + .count() + + 2 +} + +/// A char-safe prefix of a response body, for putting inside an error message. +/// +/// Slicing by byte offset traps the guest whenever a multibyte character +/// straddles the cut — and it is the failure-reporting paths that do the +/// slicing, so the trap replaces the report: the module dies before it can say +/// what went wrong, and the entity keeps whatever status it already had. The +/// same class already cost `monty_repl` its span document (ADR-0035 §15). +pub fn error_excerpt(text: &str, max_chars: usize) -> String { + text.chars().take(max_chars).collect() +} + +/// Serialized size of `value` before the field encoding — what a dropped-member +/// marker reports, so the number means the same thing everywhere. +fn json_len(value: &Value) -> usize { + serde_json::to_string(value) + .map(|json| json.len()) + .unwrap_or(usize::MAX) +} + +/// What one `"key":value,` member contributes to the stored field. +fn member_cost(key: &str, value: &Value) -> usize { + stored_json_len(value) + key.len() + 4 +} + +/// Drop whatever does not fit under the entry's `extra_json` ceiling. +/// +/// Past the ceiling the kernel replaces or externalizes the *whole* field, so +/// one oversized addition takes the per-turn facts with it. Writers that build +/// their own extras bound them by policy — which signal to sacrifice first — +/// but this is the invariant at the boundary every writer passes through, +/// including the JSONL sync path that re-materializes extras written before any +/// of those policies existed. Anything dropped leaves its size behind. +fn bound_entry_extra(extra: Value) -> Value { + if stored_json_len(&extra) <= MAX_ENTRY_EXTRA_BYTES { + return extra; + } + if !extra.is_object() { + // Nothing to sacrifice member by member; record the size it had. + return json!({ "_extra_json_dropped_bytes": json_len(&extra) }); + } + + let mut extra = extra; + if let Some(fields) = extra.as_object_mut() { + // Every member is measured once and the largest go first. Re-measuring + // the whole object per drop would be quadratic, and this runs inside a + // WASM guest on an object a corrupted transcript line can make wide. + let mut droppable: Vec<(String, usize)> = fields + .iter() + .filter(|(key, _)| { + !ENTRY_EXTRA_ESSENTIALS.contains(&key.as_str()) && !key.ends_with("_dropped_bytes") + }) + .map(|(key, value)| (key.clone(), member_cost(key, value))) + .collect(); + droppable.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + + let mut total = stored_json_len(&Value::Object(fields.clone())); + for (key, cost) in droppable { + if total <= MAX_ENTRY_EXTRA_BYTES { + break; + } + let size = fields.remove(&key).map(|value| json_len(&value)).unwrap_or(0); + let marker = format!("{key}_dropped_bytes"); + total = total.saturating_sub(cost) + member_cost(&marker, &json!(size)); + fields.insert(marker, json!(size)); + } + + // A per-turn fact can be oversized on its own — a provider is free to + // return a huge `stop_reason`, and the JSONL sync path copies whatever + // a transcript line put under that key — and nothing above would have + // touched it, because the drop loop skips the essentials. + bound_essential_values(fields); + } + + // Hard floor. An extras object with thousands of small members leaves a + // marker behind for each one, and the markers alone can hold the value over + // the ceiling. The per-turn facts are worth more than the drop record, so + // they are what survives, with a single count in place of the markers. + if stored_json_len(&extra) > MAX_ENTRY_EXTRA_BYTES + && let Some(fields) = extra.as_object_mut() + { + let before = fields.len(); + // The token-signal drop markers survive alongside the facts. They are + // the emitter's only evidence that signals existed and were refused, + // and there are at most four of them, so they cannot be what holds the + // value over — unlike the arbitrary `_dropped_bytes` markers this + // floor exists to shed. + fields.retain(|key, _| { + ENTRY_EXTRA_ESSENTIALS.contains(&key.as_str()) + || TOKEN_SIGNAL_DROP_MARKERS.contains(&key.as_str()) + }); + let removed = before - fields.len(); + bound_essential_values(fields); + fields.insert("_extra_json_dropped_members".to_string(), json!(removed)); + } + + // The invariant is unconditional: returning a value over the ceiling costs + // the entire field, which is what this function exists to prevent. Nothing + // above should be able to reach here, so if anything does, the size is the + // one fact worth keeping. + if stored_json_len(&extra) > MAX_ENTRY_EXTRA_BYTES { + extra = json!({ "_extra_json_dropped_bytes": json_len(&extra) }); + } + + extra +} + +/// Cut every member down to a size that cannot itself breach the entry ceiling, +/// recording each cut. Reached when nothing droppable is left, so it must handle +/// the per-turn facts themselves: they are small by construction only when the +/// writer is well behaved, and one of the writers copies a transcript line's +/// keys verbatim. +fn bound_essential_values(fields: &mut serde_json::Map) { + /// Generous for a provider name, a model id or a stop reason, and small + /// enough that the whole essential set fits many times over. + const MAX_ESSENTIAL_STRING_CHARS: usize = 512; + /// Anything not a short string or a scalar is not the fact this key names. + const MAX_ESSENTIAL_VALUE_BYTES: usize = 4_096; + + let oversized: Vec = fields + .iter() + .filter(|(key, value)| { + !key.ends_with("_truncated_chars") + && !key.ends_with("_dropped_bytes") + && (value + .as_str() + .is_some_and(|text| text.chars().count() > MAX_ESSENTIAL_STRING_CHARS) + || json_len(value) > MAX_ESSENTIAL_VALUE_BYTES) + }) + .map(|(key, _)| key.clone()) + .collect(); + + for key in oversized { + let Some(value) = fields.get(&key) else { + continue; + }; + match value.as_str() { + Some(text) => { + let original = text.chars().count(); + let cut: String = text.chars().take(MAX_ESSENTIAL_STRING_CHARS).collect(); + fields.insert(key.clone(), json!(cut)); + fields.insert(format!("{key}_truncated_chars"), json!(original)); + } + // An array or object under a per-turn fact's key is corrupt data, + // not the fact. Its size is the only part worth keeping. + None => { + let size = json_len(value); + fields.remove(&key); + fields.insert(format!("{key}_dropped_bytes"), json!(size)); + } + } + } +} + fn session_entry_create_body(spec: &SessionEntryCreateSpec<'_>) -> Result { let content_json = spec .content @@ -626,7 +970,7 @@ fn session_entry_create_body(spec: &SessionEntryCreateSpec<'_>) -> Result bool { .unwrap_or(false) } +/// Append one SessionEntry under `parent_entry_id`. +/// +/// `extra_json` carries per-turn facts the OTS emitter needs later (provider, +/// model, stop reason, usage, token-level RL signals). It is merged into the +/// entry's ExtraJson alongside the wall-clock stamp; pass `None` when the +/// caller has nothing to add. +#[allow(clippy::too_many_arguments)] pub fn append_session_entry_inline( ctx: &Context, temper_api_url: &str, @@ -791,6 +1142,7 @@ pub fn append_session_entry_inline( role: &str, content: &Value, tokens: usize, + extra_json: Option<&Value>, ) -> Result { let session_id = session_id_from_entries_ref(session_ref) .ok_or("append_session_entry_inline requires session-entries: ref")?; @@ -812,7 +1164,7 @@ pub fn append_session_entry_inline( Some(content), None, None, - None, + extra_json, tokens, ) } @@ -957,7 +1309,7 @@ fn list_session_entries( return Err(format!( "SessionEntry list failed (HTTP {}): {}", resp.status, - &resp.body[..resp.body.len().min(300)] + error_excerpt(&resp.body, 300) )); } let parsed: Value = serde_json::from_str(&resp.body) @@ -1140,7 +1492,7 @@ pub fn read_text_files_batch( Err(format!( "TemperFS batch read failed (HTTP {}): {}", last_status, - &last_body[..last_body.len().min(200)] + error_excerpt(&last_body, 200) )) } @@ -1195,7 +1547,7 @@ pub fn read_text_file_versions_batch( Err(format!( "TemperFS batch version read failed (HTTP {}): {}", last_status, - &last_body[..last_body.len().min(200)] + error_excerpt(&last_body, 200) )) } @@ -1348,7 +1700,7 @@ pub fn create_content_file_ref( return Err(format!( "content file creation failed (HTTP {}): {}", file_resp.status, - &file_resp.body[..file_resp.body.len().min(300)] + error_excerpt(&file_resp.body, 300) )); } @@ -1534,7 +1886,7 @@ fn read_content_file_head( return Err(format!( "content file head read failed (HTTP {}): {}", resp.status, - &resp.body[..resp.body.len().min(200)] + error_excerpt(&resp.body, 200) )); } serde_json::from_str(&resp.body).map_err(|e| format!("parse content file head response: {e}")) @@ -1587,7 +1939,7 @@ pub mod bounded_reads { use serde_json::{Value, json}; use temper_wasm_sdk::prelude::*; - use super::entity_field_str; + use super::{entity_field_str, error_excerpt}; pub const POINT_LOOKUP_TOP: usize = 20; @@ -1679,7 +2031,7 @@ pub mod bounded_reads { return Err(format!( "{label}: GET {path} failed (HTTP {}): {}", resp.status, - &resp.body[..resp.body.len().min(300)] + error_excerpt(&resp.body, 300) )); } if resp.body.is_empty() { @@ -1917,6 +2269,184 @@ pub fn parse_iso8601_to_epoch_secs(s: &str) -> Option { mod tests { use super::*; + /// Past the `extra_json` ceiling the kernel replaces or externalizes the + /// whole field, so one oversized member costs the turn its date, model and + /// token counts as well. Writers bound their own extras by policy; this is + /// the invariant every writer passes through, including the JSONL sync path + /// that re-materializes extras written before any of those policies existed. + #[test] + fn entry_extra_is_bounded_at_the_write_boundary() { + let oversized: Vec = (0..MAX_ENTRY_EXTRA_BYTES).map(|i| json!(i % 10)).collect(); + let spec = SessionEntryCreateSpec { + session_id: "ss-1", + entry_id: "a-1", + parent_entry_id: Some("u-1"), + sequence: 2, + entry_type: "message", + role: Some("assistant"), + content: None, + content_file_id: None, + content_file_version_id: None, + extra_json: Some(&json!({ + "ts_ms": 1_767_225_600_000_i64, + "provider": "anthropic", + "model": "claude-sonnet-4-6", + "input_tokens": 120, + "output_tokens": 34, + "logprobs": oversized, + })), + tokens: 34, + }; + + let body = session_entry_create_body(&spec).expect("body builds"); + let stored = body["ExtraJson"].as_str().expect("ExtraJson is a string"); + assert!( + stored_json_len(&Value::String(stored.to_string())) <= MAX_ENTRY_EXTRA_BYTES, + "the stored field must fit the ceiling the kernel measures, got {} bytes", + stored.len() + ); + + let extra: Value = serde_json::from_str(stored).expect("stored extra parses"); + assert_eq!( + extra["ts_ms"], 1_767_225_600_000_i64, + "the per-turn facts are the last thing sacrificed" + ); + assert_eq!(extra["provider"], "anthropic"); + assert_eq!(extra["output_tokens"], 34); + assert!(extra.get("logprobs").is_none()); + assert!( + extra["logprobs_dropped_bytes"].as_u64().unwrap_or(0) > 0, + "a dropped member must leave its size behind: {extra}" + ); + } + + /// The bound has to hold even when nothing is droppable. A provider is free + /// to return a huge `stop_reason`, and an extras object can carry thousands + /// of small members whose drop markers alone exceed the ceiling. Returning + /// an oversized value costs the entire field, which is the outcome the + /// bound exists to prevent, so both cases end under the ceiling. + #[test] + fn entry_extra_bound_holds_when_nothing_is_droppable() { + let huge_reason = "x".repeat(MAX_ENTRY_EXTRA_BYTES + 1024); + let bounded = bound_entry_extra(json!({ + "ts_ms": 1_767_225_600_000_i64, + "provider": "anthropic", + "stop_reason": huge_reason, + "output_tokens": 34, + })); + assert!( + stored_json_len(&bounded) <= MAX_ENTRY_EXTRA_BYTES, + "an oversized essential must be shortened, not passed through" + ); + assert_eq!(bounded["ts_ms"], 1_767_225_600_000_i64); + assert_eq!(bounded["output_tokens"], 34); + assert!(bounded["stop_reason_truncated_chars"].as_u64().unwrap() > 0); + + // Not every oversized essential is a string: the JSONL sync path copies + // a transcript line's keys verbatim, so a corrupted line can put an + // array under one. Truncation does not apply, and the drop loop skips + // essentials, so this used to sail past the bound entirely. + let bounded = bound_entry_extra(json!({ + "ts_ms": 1_767_225_600_000_i64, + "stop_reason": vec![7_u64; MAX_ENTRY_EXTRA_BYTES], + })); + assert!( + stored_json_len(&bounded) <= MAX_ENTRY_EXTRA_BYTES, + "an oversized non-string essential must be bounded too, got {} bytes", + stored_json_len(&bounded) + ); + assert_eq!(bounded["ts_ms"], 1_767_225_600_000_i64); + assert!(bounded["stop_reason_dropped_bytes"].as_u64().unwrap() > 0); + + let mut crowded = serde_json::Map::new(); + crowded.insert("ts_ms".to_string(), json!(1_767_225_600_000_i64)); + crowded.insert("model".to_string(), json!("claude-sonnet-4-6")); + for index in 0..20_000 { + crowded.insert(format!("k{index}"), json!("payload")); + } + let bounded = bound_entry_extra(Value::Object(crowded)); + assert!( + stored_json_len(&bounded) <= MAX_ENTRY_EXTRA_BYTES, + "a marker per dropped member cannot be allowed to hold the value over" + ); + assert_eq!(bounded["ts_ms"], 1_767_225_600_000_i64); + assert_eq!(bounded["model"], "claude-sonnet-4-6"); + assert!( + bounded["_extra_json_dropped_members"].as_u64().unwrap() > 0, + "the count stands in for the markers it replaced: {bounded}" + ); + } + + /// The hard floor sheds drop markers, but the token-signal ones are the + /// emitter's only evidence that signals existed and were refused — losing + /// them turns a refused signal back into "the serving stack sent none". + /// There are at most four, so they cannot be what holds the value over. + #[test] + fn entry_extra_bound_keeps_the_token_signal_drop_markers() { + let mut crowded = serde_json::Map::new(); + crowded.insert("ts_ms".to_string(), json!(1_767_225_600_000_i64)); + crowded.insert("logprobs_dropped_bytes".to_string(), json!(40_000)); + crowded.insert("completion_token_ids_dropped_bytes".to_string(), json!(35_000)); + for index in 0..20_000 { + crowded.insert(format!("k{index}"), json!("payload")); + } + + let bounded = bound_entry_extra(Value::Object(crowded)); + assert!(stored_json_len(&bounded) <= MAX_ENTRY_EXTRA_BYTES); + assert_eq!(bounded["ts_ms"], 1_767_225_600_000_i64); + assert_eq!( + bounded["logprobs_dropped_bytes"], 40_000, + "a refused signal must stay visible: {bounded}" + ); + assert_eq!(bounded["completion_token_ids_dropped_bytes"], 35_000); + assert!(bounded["_extra_json_dropped_members"].as_u64().unwrap() > 0); + } + + /// A reader that stores what it read as a record of what an agent did has + /// to be able to tell an empty transcript from a missing one. Collapsing + /// both to `Ok("")` is right for a writer resuming a tree and wrong here. + #[test] + fn transcript_presence_separates_content_from_absence() { + assert_eq!( + transcript_presence("{\"id\":\"u-1\"}\n", TranscriptPresence::EmptyFile), + TranscriptPresence::Present + ); + assert_eq!( + transcript_presence("", TranscriptPresence::EmptyFile), + TranscriptPresence::EmptyFile + ); + assert_eq!( + transcript_presence("", TranscriptPresence::NoEntries), + TranscriptPresence::NoEntries + ); + assert_eq!( + transcript_presence(" \n\n \n", TranscriptPresence::NoEntries), + TranscriptPresence::NoEntries, + "a transcript of nothing but whitespace carries no entries" + ); + } + + /// The reasons travel into the stored trajectory, so they are a wire + /// contract and cannot be renamed casually. + #[test] + fn transcript_presence_names_are_stable() { + for (presence, name) in [ + (TranscriptPresence::Present, "present"), + (TranscriptPresence::PendingFirstTurn, "pending_first_turn"), + (TranscriptPresence::NoEntries, "no_entries"), + (TranscriptPresence::MissingFile, "missing_file"), + (TranscriptPresence::EmptyFile, "empty_file"), + (TranscriptPresence::Undeclared, "undeclared"), + ] { + assert_eq!(presence.as_str(), name); + assert_eq!( + presence.is_present(), + presence == TranscriptPresence::Present, + "only Present may report as present" + ); + } + } + #[test] fn bounded_read_helper_builds_point_lookup_urls() { let filter = format!( @@ -2269,6 +2799,32 @@ mod tests { assert_eq!(lines[1005]["parentId"], "a-1004"); } + #[test] + fn stamp_recorded_at_adds_wall_clock_to_every_entry() { + let stamped = stamp_recorded_at(None, 1_767_225_600_000); + assert_eq!(stamped["ts_ms"], 1_767_225_600_000_i64); + + let existing = json!({ "version": 1 }); + let stamped = stamp_recorded_at(Some(&existing), 42); + assert_eq!(stamped["version"], 1, "caller extras must survive"); + assert_eq!(stamped["ts_ms"], 42); + } + + #[test] + fn stamp_recorded_at_preserves_an_explicit_timestamp() { + let existing = json!({ "ts_ms": 7 }); + let stamped = stamp_recorded_at(Some(&existing), 99); + assert_eq!(stamped["ts_ms"], 7, "an explicit stamp is authoritative"); + } + + #[test] + fn stamp_recorded_at_wraps_non_object_extras() { + let existing = json!("legacy"); + let stamped = stamp_recorded_at(Some(&existing), 5); + assert_eq!(stamped["extra"], "legacy"); + assert_eq!(stamped["ts_ms"], 5); + } + #[test] fn session_entry_create_body_shapes_header_and_user_entries() { let header_extra = json!({"version": 1}); diff --git a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml index 54900cce9..04beb6325 100644 --- a/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_provisioner/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml index 7ddd2e541..ca6997bd1 100644 --- a/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml +++ b/os-apps/paw-agent/wasm/workspace_restorer/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../wasm-helpers" } diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock index 518319529..b326b72ec 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml index e1fc6fbd5..37d142428 100644 --- a/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/initialize_tournament/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock index e1d39c03d..3032d7210 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml index a6b6b0613..db4e97c6b 100644 --- a/os-apps/paw-autoreason/wasm/run_round/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/run_round/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock index 84359c402..28c5d357c 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml index 690617a03..661764d4a 100644 --- a/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml +++ b/os-apps/paw-autoreason/wasm/tally_votes/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml index 1a0e34614..625db0df8 100644 --- a/os-apps/paw-channels/wasm/channel_connect/Cargo.toml +++ b/os-apps/paw-channels/wasm/channel_connect/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-channels/wasm/route_message/Cargo.toml b/os-apps/paw-channels/wasm/route_message/Cargo.toml index 415a0d623..ceaab5cf3 100644 --- a/os-apps/paw-channels/wasm/route_message/Cargo.toml +++ b/os-apps/paw-channels/wasm/route_message/Cargo.toml @@ -11,4 +11,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-channels/wasm/send_reply/Cargo.toml b/os-apps/paw-channels/wasm/send_reply/Cargo.toml index b19ee85ba..efed3a179 100644 --- a/os-apps/paw-channels/wasm/send_reply/Cargo.toml +++ b/os-apps/paw-channels/wasm/send_reply/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock index d405f3f97..c4f34f0c4 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml index d0b68a358..c84a82f54 100644 --- a/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml +++ b/os-apps/paw-channels/wasm/transport_reconcile/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock index 75ad007e0..f23ff7e8a 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml index 06e9b9986..cf399a4dd 100644 --- a/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml +++ b/os-apps/paw-consilium/wasm/check_and_synthesize/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock index 83e24e47f..a857f9c47 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml index 52a732f92..65a4cbe40 100644 --- a/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml +++ b/os-apps/paw-consilium/wasm/spawn_perspectives/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/adjudicate_nodes/Cargo.toml b/os-apps/paw-foresight/wasm/adjudicate_nodes/Cargo.toml index 446b585f1..71b5b32d0 100644 --- a/os-apps/paw-foresight/wasm/adjudicate_nodes/Cargo.toml +++ b/os-apps/paw-foresight/wasm/adjudicate_nodes/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.lock b/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.lock index 74db553e3..2010226da 100644 --- a/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.lock +++ b/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.toml b/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.toml index 09f707c83..f49a837b4 100644 --- a/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.toml +++ b/os-apps/paw-foresight/wasm/aggregate_costs/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/animate_dwellers/Cargo.toml b/os-apps/paw-foresight/wasm/animate_dwellers/Cargo.toml index 2ba7369eb..899e122f5 100644 --- a/os-apps/paw-foresight/wasm/animate_dwellers/Cargo.toml +++ b/os-apps/paw-foresight/wasm/animate_dwellers/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/consistency_gate/Cargo.lock b/os-apps/paw-foresight/wasm/consistency_gate/Cargo.lock index 635a247c0..707ee36c1 100644 --- a/os-apps/paw-foresight/wasm/consistency_gate/Cargo.lock +++ b/os-apps/paw-foresight/wasm/consistency_gate/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/consistency_gate/Cargo.toml b/os-apps/paw-foresight/wasm/consistency_gate/Cargo.toml index a32ef430b..88c8078dc 100644 --- a/os-apps/paw-foresight/wasm/consistency_gate/Cargo.toml +++ b/os-apps/paw-foresight/wasm/consistency_gate/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/decompose_endpoint/Cargo.toml b/os-apps/paw-foresight/wasm/decompose_endpoint/Cargo.toml index 511ea4867..42beeaf67 100644 --- a/os-apps/paw-foresight/wasm/decompose_endpoint/Cargo.toml +++ b/os-apps/paw-foresight/wasm/decompose_endpoint/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.lock b/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.lock index 9f9375e5d..3a113c5a8 100644 --- a/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.lock +++ b/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.toml b/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.toml index 3aa673a8e..ea5e8199f 100644 --- a/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.toml +++ b/os-apps/paw-foresight/wasm/evidence_ingest/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.lock b/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.lock index 7a31fb037..0bdb202ef 100644 --- a/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.lock +++ b/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.toml b/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.toml index 715c31dcb..2e9267744 100644 --- a/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.toml +++ b/os-apps/paw-foresight/wasm/grade_hindcast/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" corridor-embed = { path = "../corridor_embed" } diff --git a/os-apps/paw-foresight/wasm/register_forecasts/Cargo.lock b/os-apps/paw-foresight/wasm/register_forecasts/Cargo.lock index aa59cb6bc..142ed3161 100644 --- a/os-apps/paw-foresight/wasm/register_forecasts/Cargo.lock +++ b/os-apps/paw-foresight/wasm/register_forecasts/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/register_forecasts/Cargo.toml b/os-apps/paw-foresight/wasm/register_forecasts/Cargo.toml index 70039df28..16b9f32d1 100644 --- a/os-apps/paw-foresight/wasm/register_forecasts/Cargo.toml +++ b/os-apps/paw-foresight/wasm/register_forecasts/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" corridor-embed = { path = "../corridor_embed" } diff --git a/os-apps/paw-foresight/wasm/render_artifacts/Cargo.lock b/os-apps/paw-foresight/wasm/render_artifacts/Cargo.lock index 9dc59f899..1bb432de8 100644 --- a/os-apps/paw-foresight/wasm/render_artifacts/Cargo.lock +++ b/os-apps/paw-foresight/wasm/render_artifacts/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/render_artifacts/Cargo.toml b/os-apps/paw-foresight/wasm/render_artifacts/Cargo.toml index 2bcba1e69..bc0962a23 100644 --- a/os-apps/paw-foresight/wasm/render_artifacts/Cargo.toml +++ b/os-apps/paw-foresight/wasm/render_artifacts/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.lock b/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.lock index 2c703aff5..a5b140983 100644 --- a/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.lock +++ b/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.toml b/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.toml index 0382dd4fe..29d20a69e 100644 --- a/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.toml +++ b/os-apps/paw-foresight/wasm/sample_endpoints/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" corridor-embed = { path = "../corridor_embed" } diff --git a/os-apps/paw-foresight/wasm/seed_world/Cargo.lock b/os-apps/paw-foresight/wasm/seed_world/Cargo.lock index 2c522e8f1..41fde2527 100644 --- a/os-apps/paw-foresight/wasm/seed_world/Cargo.lock +++ b/os-apps/paw-foresight/wasm/seed_world/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/seed_world/Cargo.toml b/os-apps/paw-foresight/wasm/seed_world/Cargo.toml index 2fd6de702..472e92c79 100644 --- a/os-apps/paw-foresight/wasm/seed_world/Cargo.toml +++ b/os-apps/paw-foresight/wasm/seed_world/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.lock index 3c483d098..b4d446a55 100644 --- a/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.toml index ac0538adb..2f914bf00 100644 --- a/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_adversaries/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.lock b/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.lock index 5c1cb9084..91b232a53 100644 --- a/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.lock +++ b/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.toml b/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.toml index 67041fdeb..f2b63a35a 100644 --- a/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.toml +++ b/os-apps/paw-foresight/wasm/spawn_repairers/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml index 1cd7d87d4..4f857b5c5 100644 --- a/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml +++ b/os-apps/paw-fs/wasm/artifact_batch_apply/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml index 97c6ba1f3..738b4e732 100644 --- a/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml +++ b/os-apps/paw-fs/wasm/workspace_fs/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml index 034c9fa69..db2829c5d 100644 --- a/os-apps/paw-heal/wasm/alert_opener/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_opener/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml index 1b7bf2e7e..a4b8bd2ff 100644 --- a/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml +++ b/os-apps/paw-heal/wasm/alert_verifier/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml index aaef700cb..eb8943b67 100644 --- a/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_initiator/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml index c8893ae0c..7fb764af4 100644 --- a/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml +++ b/os-apps/paw-heal/wasm/cicd_merger/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml index 6d1fe413b..ca2307b10 100644 --- a/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml +++ b/os-apps/paw-heal/wasm/deployment_tracker/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml index 1b16a65f7..10a2a3fc3 100644 --- a/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml +++ b/os-apps/paw-heal/wasm/heal_reporter/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml index dfea4f616..260f9fa27 100644 --- a/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/process_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml index fc6f75779..279fd6ee4 100644 --- a/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/route_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml index 2b2c22f55..506ba91c0 100644 --- a/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml +++ b/os-apps/paw-ingest/wasm/validate_webhook/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock index 5c823eb70..04128bc72 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml index 01baf5cd7..433967850 100644 --- a/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/event_emitter/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } session-tree-lib = { path = "../../../paw-agent/wasm/session-tree-lib" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock index 988602294..5164be4f5 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml index beb6df565..f34bf776b 100644 --- a/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/managed_agent_updater/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock index 4f2ff07f4..057ee2020 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml index 19711325f..daca3b4db 100644 --- a/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_orchestrator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock index 5192a6dfe..cf0605138 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml index 1b37df820..0303e3ea1 100644 --- a/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml +++ b/os-apps/paw-managed-agents/wasm/session_terminator/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-media/wasm/openai_codex_image_generate/Cargo.toml b/os-apps/paw-media/wasm/openai_codex_image_generate/Cargo.toml index af7232ceb..f3543517f 100644 --- a/os-apps/paw-media/wasm/openai_codex_image_generate/Cargo.toml +++ b/os-apps/paw-media/wasm/openai_codex_image_generate/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } openai-codex-wire = { path = "../../../paw-agent/wasm/openai-codex-wire" } base64 = "0.22" diff --git a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml index 383a70d3c..f97e755c6 100644 --- a/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/daily_brief_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml index 6e6486e31..0ac251344 100644 --- a/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/finding_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml index 7cf803871..03236b6b2 100644 --- a/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_request_router/Cargo.toml @@ -11,4 +11,4 @@ test = false [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml index 8aea19c08..46b585b90 100644 --- a/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml index 49779b361..afed06a08 100644 --- a/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/patrol_schedule_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml index c9cd8e92d..182ef169d 100644 --- a/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/repo_sweep_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml index ec8094964..31ddabfe9 100644 --- a/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/review_gate_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml index de67f423d..442831708 100644 --- a/os-apps/paw-patrol/wasm/signal_router/Cargo.toml +++ b/os-apps/paw-patrol/wasm/signal_router/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml index 0d7d70d60..e3a771660 100644 --- a/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/work_cycle_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml index 19b00e648..cf7eb1ddf 100644 --- a/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml +++ b/os-apps/paw-patrol/wasm/worker_run_lifecycle/Cargo.toml @@ -9,4 +9,4 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } diff --git a/os-apps/paw-research/wasm/web_fetch/Cargo.toml b/os-apps/paw-research/wasm/web_fetch/Cargo.toml index 0f5b254a1..1f3346e0a 100644 --- a/os-apps/paw-research/wasm/web_fetch/Cargo.toml +++ b/os-apps/paw-research/wasm/web_fetch/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-research/wasm/web_search/Cargo.toml b/os-apps/paw-research/wasm/web_search/Cargo.toml index 189c01593..a2ff14554 100644 --- a/os-apps/paw-research/wasm/web_search/Cargo.toml +++ b/os-apps/paw-research/wasm/web_search/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml index 2e036b4d7..6b6be8164 100644 --- a/os-apps/paw-skills/wasm/skill_installer/Cargo.toml +++ b/os-apps/paw-skills/wasm/skill_installer/Cargo.toml @@ -12,5 +12,5 @@ crate-type = ["cdylib"] hex = "0.4" serde_json = "1" sha2 = "0.10" -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } wasm-helpers = { path = "../../../paw-agent/wasm/wasm-helpers" } diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock index e524f7a25..102cefa24 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml index e7fb52a01..e4d99b9b6 100644 --- a/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml +++ b/os-apps/paw-wiki/wasm/build_session_message/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock index 5c982db39..d7ce412fa 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.lock @@ -97,7 +97,7 @@ dependencies = [ [[package]] name = "temper-wasm-sdk" version = "0.1.0" -source = "git+https://github.com/nerdsane/temper.git?rev=804633e2c5cab3b0bd334f78bfb5ea23aca1858d#804633e2c5cab3b0bd334f78bfb5ea23aca1858d" +source = "git+https://github.com/nerdsane/temper.git?rev=a747f7d40cb556371168f8460bc72806c3574d2b#a747f7d40cb556371168f8460bc72806c3574d2b" dependencies = [ "serde", "serde_json", diff --git a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml index 82040e9e1..20c4f217e 100644 --- a/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml +++ b/os-apps/paw-wiki/wasm/finalize_spawned_session/Cargo.toml @@ -9,5 +9,5 @@ crate-type = ["cdylib"] [workspace] [dependencies] -temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "804633e2c5cab3b0bd334f78bfb5ea23aca1858d" } +temper-wasm-sdk = { git = "https://github.com/nerdsane/temper.git", rev = "a747f7d40cb556371168f8460bc72806c3574d2b" } serde_json = "1" diff --git a/scripts/prove_track3_ots.py b/scripts/prove_track3_ots.py index b31909467..608aba973 100644 --- a/scripts/prove_track3_ots.py +++ b/scripts/prove_track3_ots.py @@ -49,6 +49,27 @@ def wait_for_terminal(client: ODataClient, session_id: str, timeout_s: float) -> ) +# Statuses `emit_ots_trajectory` records when it is done with a session. +SETTLED_EMISSION_STATUSES = {"emitted", "emitted_degraded", "failed"} + + +def wait_for_emission(client: ODataClient, session_id: str, timeout_s: float) -> dict: + """Re-read the Session until the emitter has recorded its outcome. + + The terminal transition only *triggers* emission; the guest runs after it. + Reading the entity the moment the status turns terminal races that, and the + race reads as "no trajectory was emitted" rather than "not yet". + """ + deadline = time.time() + timeout_s + fields: dict = {} + while time.time() < deadline: + fields = client.get("Sessions", session_id).get("fields", {}) + if (fields.get("trajectory_emission_status") or "") in SETTLED_EMISSION_STATUSES: + return fields + time.sleep(2) + return fields + + def main() -> None: parser = argparse.ArgumentParser(description="Prove Track 3 OTS emission end-to-end.") parser.add_argument( @@ -65,7 +86,25 @@ def main() -> None: ) parser.add_argument( "--user-message", - default="List the files in /workspace and then call temper.done(\"ok\").", + default=( + "Use the temper.list tool once to list up to 3 Sessions, then reply " + "with how many you saw. Do not call any other tool." + ), + ) + parser.add_argument( + "--model", + default=os.environ.get("LLM_MODEL", ""), + help="Provider model; defaults to LLM_MODEL from the environment.", + ) + parser.add_argument( + "--provider", + default=os.environ.get("LLM_PROVIDER", ""), + help="Provider id; defaults to LLM_PROVIDER from the environment.", + ) + parser.add_argument( + "--tools-enabled", + default="temper_list,temper_get", + help="Tools the session may call. At least one is needed for decisions.", ) parser.add_argument( "--timeout-s", @@ -96,14 +135,30 @@ def main() -> None: # ── Step 2: Create a Session ──────────────────────────────── print("[2/6] Creating a Session...") - session = client.create("Sessions", {"user_message": args.user_message}) + session = client.create("Sessions", {}) sid = entity_id(session) print(f" PASS: Session entity_id={sid}") - # ── Step 3: Start it ──────────────────────────────────────── - print("[3/6] Dispatching TemperPaw.Start...") - client.action("Sessions", sid, "TemperPaw.Start", {}) - print(" PASS: Start action accepted") + # ── Step 3: Configure it ──────────────────────────────────── + # Same entry point production uses: `route_message` creates a blank Session + # and dispatches TemperPaw.Configure, which schedules ProvisionWorkspace + # itself. There is no Start action on the Session automaton. + print("[3/6] Dispatching TemperPaw.Configure...") + client.action( + "Sessions", + sid, + "TemperPaw.Configure", + { + "user_message": args.user_message, + "model": args.model, + "provider": args.provider, + "tools_enabled": args.tools_enabled, + "temper_api_url": args.base_url, + "max_turns": "6", + "session_mode": "execute", + }, + ) + print(" PASS: Configure action accepted") # ── Step 4: Wait for terminal state ───────────────────────── print(f"[4/6] Waiting up to {args.timeout_s:.0f}s for a terminal state...") @@ -119,7 +174,9 @@ def main() -> None: # ── Step 5: Verify Phase 1 + Phase 2 fields on the entity ─── print("[5/6] Verifying OTS-related entity fields...") - fields = final_session.get("fields", final_session) + fields = wait_for_emission(client, sid, 60.0) or final_session.get( + "fields", final_session + ) tool_spans_file_id = fields.get("tool_spans_file_id") or "" trajectory_id = fields.get("trajectory_id") or "" emission_status = fields.get("trajectory_emission_status") or "" @@ -133,7 +190,12 @@ def main() -> None: for desc, ok in checks.items(): print(f" {'PASS' if ok else 'FAIL'}: {desc}") if not all(checks.values()): - print(f" emission_error='{emission_error}' (if populated, indicates POST failed)") + if emission_status == "emitted_degraded": + # The row exists but was built without some of its evidence; the + # entity names which piece, so the proof does not have to guess. + print(f" degraded emission: missing {emission_error}") + else: + print(f" emission_error='{emission_error}' (if populated, indicates POST failed)") sys.exit(3) # ── Step 6: GET /api/ots/trajectories and match by trajectory_id ──