diff --git a/.proofs/2026-06-18-pawfs-direct-key-write-hotfix.md b/.proofs/2026-06-18-pawfs-direct-key-write-hotfix.md new file mode 100644 index 00000000..0e2d84a6 --- /dev/null +++ b/.proofs/2026-06-18-pawfs-direct-key-write-hotfix.md @@ -0,0 +1,57 @@ +# PawFS direct-key write hotfix proof + +Date: 2026-06-18 + +## Scope + +Live Katagami regeneration reached `temper.write`, then failed on PawFS directory +resolution because Monty looked up directories with a broad OData collection +filter: + +`/tdata/Directories?$filter=Path eq ... and WorkspaceId eq ...` + +Production rejected that shape with `413 QueryTooLarge` once the directory table +was large enough. This patch keeps the existing PawFS entity model and changes +Monty write resolution to derive deterministic directory/file IDs from +`(workspace_id, normalized_path)`, then read by key before creating missing +entities. + +## ADR judgement + +No new ADR. This is a scoped implementation repair under the existing PawFS +direct-access model, not a new entity type, workflow, storage model, trigger, +policy boundary, or agent capability surface. + +## Red + +Added `monty_pawfs_write_path_uses_direct_keys_not_broad_path_filters` in +`crates/temperpaw/tests/paw_fs_hot_path.rs`. + +Initial result before implementation: + +```text +cargo test -p temperpaw --test paw_fs_hot_path monty_pawfs_write_path_uses_direct_keys_not_broad_path_filters +test monty_pawfs_write_path_uses_direct_keys_not_broad_path_filters ... FAILED +Monty PawFS writes should derive deterministic directory ids +``` + +## Green + +Local verification after implementation: + +```text +cargo test -p temperpaw --test paw_fs_hot_path +test result: ok. 14 passed; 0 failed + +cargo test --manifest-path os-apps/paw-agent/wasm/monty_repl/Cargo.toml +test result: ok. 73 passed; 0 failed + +./os-apps/paw-agent/wasm/build.sh +All WASM modules built, including rebuilt monty_repl. +``` + +## Live E2E + +Pending. Next step is to publish the new `temperpaw/paw-agent` Genesis ref, +update/reinstall OpenPaw production, and rerun the live Katagami regeneration +job that reproduced the PawFS write-path 413. diff --git a/crates/temperpaw/tests/paw_fs_hot_path.rs b/crates/temperpaw/tests/paw_fs_hot_path.rs index 1495e8bc..85538090 100644 --- a/crates/temperpaw/tests/paw_fs_hot_path.rs +++ b/crates/temperpaw/tests/paw_fs_hot_path.rs @@ -104,6 +104,32 @@ fn monty_exposes_write_many_for_artifact_sets() { ); } +#[test] +fn monty_pawfs_write_path_uses_direct_keys_not_broad_path_filters() { + let source = repo_file("os-apps/paw-agent/wasm/monty_repl/src/entity_ops.rs"); + + assert!( + source.contains("pawfs_directory_id"), + "Monty PawFS writes should derive deterministic directory ids" + ); + assert!( + source.contains("pawfs_file_id"), + "Monty PawFS writes should derive deterministic file ids" + ); + assert!( + source.contains("/tdata/Directories('{directory_id}')"), + "Monty PawFS writes should read directories by key" + ); + assert!( + source.contains("/tdata/Files('{file_id}')"), + "Monty PawFS writes should read files by key" + ); + assert!( + !source.contains("Path eq '{}' and WorkspaceId eq '{}' and Status ne 'Archived'"), + "Monty PawFS writes must not use broad Path+Workspace collection filters that hit bounded OData candidate limits" + ); +} + #[test] fn default_agent_tool_allowlists_include_write_many() { for path in [ diff --git a/crates/temperpaw/tests/session_turn_architecture.rs b/crates/temperpaw/tests/session_turn_architecture.rs index b45e1b7d..26c90fda 100644 --- a/crates/temperpaw/tests/session_turn_architecture.rs +++ b/crates/temperpaw/tests/session_turn_architecture.rs @@ -359,6 +359,10 @@ fn session_entry_readbacks_stay_within_bounded_query_budget() { let root = repo_root(); let helpers = fs::read_to_string(root.join("os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs")) .expect("wasm helpers source should exist"); + let context_preparer_wasm = + fs::read(root.join("os-apps/paw-agent/wasm/context_preparer/context_preparer.wasm")) + .expect("context_preparer.wasm should be checked in"); + let context_preparer_wasm = String::from_utf8_lossy(&context_preparer_wasm); let route_message = fs::read_to_string(root.join("os-apps/paw-channels/wasm/route_message/src/lib.rs")) .expect("route_message source should exist"); @@ -375,6 +379,32 @@ fn session_entry_readbacks_stay_within_bounded_query_budget() { helpers.contains("session_entries_verify_urls"), "batched SessionEntry readback should use one bounded per-entry URL per expected entry" ); + assert!( + helpers.contains("read_session_entries_from_leaf"), + "SessionEntry history reads should prefer bounded direct EntryId parent-chain reads from session_leaf_id" + ); + assert!( + helpers.contains("ParentEntryId"), + "SessionEntry parent-chain reads must follow ParentEntryId instead of session-wide scans" + ); + assert!( + context_preparer_wasm.contains("SessionEntry direct lookup failed") + && context_preparer_wasm.contains("SessionEntry parent-chain read") + && context_preparer_wasm.contains("SessionEntry virtual first-turn leaf"), + "built context_preparer.wasm must include the bounded leaf parent-chain reader" + ); + assert!( + helpers.contains("$orderby=Sequence%20asc"), + "SessionEntry history reads should use deterministic keyset ordering" + ); + assert!( + helpers.contains("Sequence%20ge%20{next_sequence}"), + "SessionEntry history reads should continue by Sequence instead of broad skip scans" + ); + assert!( + !helpers.contains("&$skip="), + "SessionEntry history reads must not use $skip because production bounded OData can reject high-candidate scans" + ); assert!( route_message.contains("session_leaf_id is missing; starting clean continuation"), "route_message should start cleanly instead of broad-scanning when the prior leaf hint is missing" diff --git a/os-apps/paw-agent/wasm/monty_repl/src/entity_ops.rs b/os-apps/paw-agent/wasm/monty_repl/src/entity_ops.rs index 3cf8d554..cc7e986b 100644 --- a/os-apps/paw-agent/wasm/monty_repl/src/entity_ops.rs +++ b/os-apps/paw-agent/wasm/monty_repl/src/entity_ops.rs @@ -2176,6 +2176,30 @@ fn http_get( .map_err(|e| format!("failed to parse response from {path}: {e}")) } +fn http_get_optional( + ctx: &Context, + api_url: &str, + _tenant: &str, + _principal_id: &str, + path: &str, +) -> Result, String> { + let url = format!("{api_url}{path}"); + let headers = internal_headers(); + let resp = ctx.http_call("GET", &url, &headers, "")?; + if let Some(denial) = dispatch::check_cedar_denial(resp.status, &resp.body) { + return Err(denial); + } + if resp.status == 404 { + return Ok(None); + } + if resp.status >= 400 { + return Err(format!("HTTP GET {path}: {} {}", resp.status, resp.body)); + } + serde_json::from_str(&resp.body) + .map(Some) + .map_err(|e| format!("failed to parse response from {path}: {e}")) +} + fn http_post( ctx: &Context, api_url: &str, @@ -2405,12 +2429,25 @@ fn pawfs_path_segments(path: &str) -> Vec<&str> { .collect() } -fn pawfs_filter_path_and_workspace(path: &str, ws_id: &str) -> String { - format!( - "Path eq '{}' and WorkspaceId eq '{}' and Status ne 'Archived'", - escape_odata_string(path), - escape_odata_string(ws_id) - ) +fn pawfs_stable_hash(parts: &[&str]) -> u64 { + let mut hash = 0xcbf29ce484222325_u64; + for part in parts { + for byte in part.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash ^= 0xff; + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +fn pawfs_directory_id(ws_id: &str, path: &str) -> String { + format!("dir-{:016x}", pawfs_stable_hash(&[ws_id, path])) +} + +fn pawfs_file_id(ws_id: &str, path: &str) -> String { + format!("fl-{:016x}", pawfs_stable_hash(&[ws_id, path])) } fn pawfs_filter_parent_and_workspace(parent_id: &str, ws_id: &str) -> String { @@ -2421,8 +2458,11 @@ fn pawfs_filter_parent_and_workspace(parent_id: &str, ws_id: &str) -> String { ) } -fn pawfs_first_entity(resp: &Value) -> Option<&Value> { - resp.get("value").and_then(Value::as_array)?.first() +fn pawfs_entity_is_archived(value: &Value) -> bool { + matches!( + entity_field_str_any(value, &["Status", "status"]), + Some("Archived") + ) } fn pawfs_entity_id(value: &Value) -> Option<&str> { @@ -2514,16 +2554,22 @@ fn find_pawfs_directory( raw_path: &str, ) -> Result, String> { let path = pawfs_normalize_path(raw_path)?; - let filter = urlenc(&pawfs_filter_path_and_workspace(&path, ws_id)); + let directory_id = pawfs_directory_id(ws_id, &path); let eid = ctx_entity_id(ctx); - let resp = http_get( + let Some(resp) = http_get_optional( ctx, api_url, tenant, eid, - &format!("/tdata/Directories?$filter={filter}"), - )?; - Ok(pawfs_first_entity(&resp).and_then(pawfs_directory_from_value)) + &format!("/tdata/Directories('{directory_id}')"), + )? + else { + return Ok(None); + }; + if pawfs_entity_is_archived(&resp) { + return Ok(None); + } + Ok(pawfs_directory_from_value(&resp)) } fn find_pawfs_file( @@ -2534,16 +2580,22 @@ fn find_pawfs_file( raw_path: &str, ) -> Result, String> { let path = pawfs_normalize_path(raw_path)?; - let filter = urlenc(&pawfs_filter_path_and_workspace(&path, ws_id)); + let file_id = pawfs_file_id(ws_id, &path); let eid = ctx_entity_id(ctx); - let resp = http_get( + let Some(resp) = http_get_optional( ctx, api_url, tenant, eid, - &format!("/tdata/Files?$filter={filter}"), - )?; - Ok(pawfs_first_entity(&resp).and_then(pawfs_file_from_value)) + &format!("/tdata/Files('{file_id}')"), + )? + else { + return Ok(None); + }; + if pawfs_entity_is_archived(&resp) { + return Ok(None); + } + Ok(pawfs_file_from_value(&resp)) } fn create_pawfs_directory( @@ -2557,6 +2609,7 @@ fn create_pawfs_directory( parent_id: Option<&str>, ) -> Result { let mut body = json!({ + "Id": pawfs_directory_id(ws_id, path), "Name": name, "Path": path, "WorkspaceId": ws_id, @@ -2665,6 +2718,7 @@ fn ensure_pawfs_file( let (dir_path, filename) = pawfs_parse_file_path(&normalized)?; let directory = ensure_pawfs_directory(ctx, api_url, tenant, principal_id, ws_id, dir_path)?; let body = json!({ + "Id": pawfs_file_id(ws_id, &normalized), "Name": filename, "Path": normalized, "DirectoryId": directory.id.clone(), @@ -2988,11 +3042,26 @@ mod tests { } #[test] - fn pawfs_filter_escapes_odata_string_literals() { + fn pawfs_deterministic_ids_are_workspace_and_path_scoped() { + let ws_one = "ws-1"; + let ws_two = "ws-2"; + let path = pawfs_normalize_path("//notes///readme.md").unwrap(); + + assert_eq!( + pawfs_directory_id(ws_one, &path), + pawfs_directory_id(ws_one, "/notes/readme.md") + ); + assert_ne!( + pawfs_directory_id(ws_one, &path), + pawfs_directory_id(ws_two, &path) + ); assert_eq!( - pawfs_filter_path_and_workspace("/notes/we're-here.md", "ws'1"), - "Path eq '/notes/we''re-here.md' and WorkspaceId eq 'ws''1' and Status ne 'Archived'" + pawfs_file_id(ws_one, &path), + pawfs_file_id(ws_one, "/notes/readme.md") ); + assert_ne!(pawfs_directory_id(ws_one, &path), pawfs_file_id(ws_one, &path)); + assert!(pawfs_directory_id(ws_one, &path).starts_with("dir-")); + assert!(pawfs_file_id(ws_one, &path).starts_with("fl-")); } #[test] diff --git a/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs b/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs index 56267ceb..da9abb12 100644 --- a/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs +++ b/os-apps/paw-agent/wasm/wasm-helpers/src/lib.rs @@ -14,8 +14,10 @@ pub const SESSION_ENTRIES_REF_PREFIX: &str = "session-entries:"; const TEMPERFS_READ_ATTEMPTS: usize = 10; const TEMPERFS_WRITE_ATTEMPTS: usize = 5; const TEMPERFS_BATCH_READ_ATTEMPTS: usize = 3; -const SESSION_ENTRIES_PAGE_SIZE: usize = 1000; -const SESSION_ENTRIES_MAX_PAGES: usize = 100; +const SESSION_ENTRIES_PAGE_SIZE: usize = 200; +const SESSION_ENTRIES_MIN_PAGE_SIZE: usize = 10; +const SESSION_ENTRIES_MAX_ENTRIES: usize = 10_000; +const SESSION_ENTRIES_MAX_CHAIN_DEPTH: usize = 10_000; #[derive(Debug, Clone, PartialEq, Eq)] pub struct BatchTextFileReadItem { @@ -718,10 +720,10 @@ fn session_entries_list_url( temper_api_url: &str, session_id: &str, top: usize, - skip: usize, + next_sequence: i64, ) -> String { format!( - "{temper_api_url}/tdata/SessionEntries?$filter=SessionId%20eq%20%27{}%27&$top={top}&$skip={skip}", + "{temper_api_url}/tdata/SessionEntries?$filter=SessionId%20eq%20%27{}%27%20and%20Sequence%20ge%20{next_sequence}&$orderby=Sequence%20asc&$top={top}", session_id.replace('\'', "''"), ) } @@ -779,7 +781,11 @@ pub fn read_session_from_entries( fields: &Value, session_id: &str, ) -> Result { - let entries = list_session_entries(ctx, temper_api_url, tenant, fields, session_id)?; + let entries = + match read_session_entries_from_leaf(ctx, temper_api_url, tenant, fields, session_id)? { + Some(entries) => entries, + None => list_session_entries(ctx, temper_api_url, tenant, fields, session_id)?, + }; Ok(session_entries_jsonl_from_entities(&entries)) } @@ -902,13 +908,27 @@ fn list_session_entries( ) -> Result, String> { let headers = runtime_headers(ctx, tenant, fields, None, Some("application/json")); let mut entries = Vec::new(); - let mut skip = 0usize; + let mut next_sequence = 0_i64; + let mut page_size = SESSION_ENTRIES_PAGE_SIZE; - for _ in 0..SESSION_ENTRIES_MAX_PAGES { - let url = - session_entries_list_url(temper_api_url, session_id, SESSION_ENTRIES_PAGE_SIZE, skip); + while entries.len() < SESSION_ENTRIES_MAX_ENTRIES { + let remaining = SESSION_ENTRIES_MAX_ENTRIES.saturating_sub(entries.len()); + let top = page_size.min(remaining); + let url = session_entries_list_url(temper_api_url, session_id, top, next_sequence); let resp = ctx.http_call("GET", &url, &headers, "")?; if resp.status != 200 { + if session_entry_query_too_large(resp.status, &resp.body) + && page_size > SESSION_ENTRIES_MIN_PAGE_SIZE + { + page_size = (page_size / 4).max(SESSION_ENTRIES_MIN_PAGE_SIZE); + ctx.log( + "warn", + &format!( + "SessionEntry list page was too broad for SessionId={session_id}; retrying with page_size={page_size}" + ), + ); + continue; + } return Err(format!( "SessionEntry list failed (HTTP {}): {}", resp.status, @@ -923,19 +943,151 @@ fn list_session_entries( .cloned() .unwrap_or_default(); let page_len = page.len(); + if page_len == 0 { + return Ok(entries); + } + let mut max_sequence: Option = None; + for entry in page.iter() { + let sequence = entity_field_i64(entry, &["Sequence", "sequence"]).ok_or_else(|| { + format!( + "SessionEntry list page for SessionId={session_id} contained an entry without Sequence" + ) + })?; + max_sequence = Some(max_sequence.map_or(sequence, |max| max.max(sequence))); + } entries.append(&mut page); - if page_len < SESSION_ENTRIES_PAGE_SIZE { + let Some(max_sequence) = max_sequence else { + return Ok(entries); + }; + let next = max_sequence.saturating_add(1); + if next <= next_sequence { + return Err(format!( + "SessionEntry list made no keyset progress for SessionId={session_id} at Sequence={next_sequence}" + )); + } + next_sequence = next; + if page_len < top { return Ok(entries); } - skip += page_len; } Err(format!( - "SessionEntry list exceeded {} pages for session {session_id}; refusing incomplete context", - SESSION_ENTRIES_MAX_PAGES + "SessionEntry list exceeded {} entries for session {session_id}; refusing incomplete context", + SESSION_ENTRIES_MAX_ENTRIES )) } +fn read_session_entries_from_leaf( + ctx: &Context, + temper_api_url: &str, + tenant: &str, + fields: &Value, + session_id: &str, +) -> Result>, String> { + let Some(leaf_id) = entity_field_str(fields, &["session_leaf_id", "SessionLeafId"]) + .filter(|value| !value.is_empty()) + else { + return Ok(None); + }; + + if !session_entries_materialized(fields) && is_virtual_first_turn_leaf(session_id, leaf_id) { + ctx.log( + "info", + &format!( + "SessionEntry virtual first-turn leaf for SessionId={session_id}; skipping direct lookup" + ), + ); + return Ok(Some(Vec::new())); + } + + let headers = runtime_headers(ctx, tenant, fields, None, Some("application/json")); + let mut current_entry_id = leaf_id.to_string(); + let mut reversed_entries = Vec::new(); + let mut visited = std::collections::BTreeSet::new(); + + while reversed_entries.len() < SESSION_ENTRIES_MAX_CHAIN_DEPTH { + if !visited.insert(current_entry_id.clone()) { + return Err(format!( + "SessionEntry parent chain for SessionId={session_id} contains a cycle at EntryId={current_entry_id}" + )); + } + + let entry = + read_session_entry_by_id(ctx, temper_api_url, &headers, session_id, ¤t_entry_id)?; + let Some(entry) = entry else { + if reversed_entries.is_empty() && !session_entries_materialized(fields) { + return Ok(Some(Vec::new())); + } + return Err(format!( + "SessionEntry parent-chain read for SessionId={session_id} could not find EntryId={current_entry_id}" + )); + }; + + let parent_entry_id = + entity_field_str(&entry, &["ParentEntryId", "parent_entry_id"]).unwrap_or(""); + let next_entry_id = parent_entry_id.to_string(); + reversed_entries.push(entry); + if next_entry_id.is_empty() { + reversed_entries.reverse(); + return Ok(Some(reversed_entries)); + } + current_entry_id = next_entry_id; + } + + Err(format!( + "SessionEntry parent chain exceeded {SESSION_ENTRIES_MAX_CHAIN_DEPTH} entries for SessionId={session_id}; refusing incomplete context" + )) +} + +fn read_session_entry_by_id( + ctx: &Context, + temper_api_url: &str, + headers: &[(String, String)], + session_id: &str, + entry_id: &str, +) -> Result, String> { + let url = session_entry_verify_url(temper_api_url, session_id, entry_id); + let resp = ctx.http_call("GET", &url, headers, "")?; + if resp.status != 200 { + return Err(format!( + "SessionEntry direct lookup failed for SessionId={session_id} EntryId={entry_id} (HTTP {}): {}", + resp.status, + &resp.body[..resp.body.len().min(300)] + )); + } + let parsed: Value = serde_json::from_str(&resp.body) + .map_err(|err| format!("parse SessionEntry direct lookup response: {err}"))?; + Ok(parsed + .get("value") + .and_then(Value::as_array) + .and_then(|items| items.first()) + .cloned()) +} + +fn session_entries_materialized(fields: &Value) -> bool { + ["session_entries_materialized", "SessionEntriesMaterialized"] + .iter() + .find_map(|key| { + fields + .get(*key) + .or_else(|| fields.get("fields").and_then(|inner| inner.get(*key))) + .and_then(boolish_json) + }) + .unwrap_or(true) +} + +fn is_virtual_first_turn_leaf(session_id: &str, leaf_id: &str) -> bool { + leaf_id == format!("u-{session_id}-0") +} + +fn session_entry_query_too_large(status: u16, body: &str) -> bool { + if status == 413 { + return true; + } + let lower = body.to_ascii_lowercase(); + lower.contains("querytoolarge") || lower.contains("bounded odata read budget") +} + fn session_entry_entity_to_jsonl(entry: &Value) -> Option { let entry_id = entity_field_str(entry, &["EntryId", "entry_id"])?; let parent_entry_id = @@ -1860,13 +2012,25 @@ mod tests { } #[test] - fn session_entries_list_url_pages_with_skip() { + fn session_entries_list_url_pages_by_sequence() { assert_eq!( - session_entries_list_url("http://temper", "ss-1", 1000, 2000), - "http://temper/tdata/SessionEntries?$filter=SessionId%20eq%20%27ss-1%27&$top=1000&$skip=2000" + session_entries_list_url("http://temper", "ss-1", 200, 40), + "http://temper/tdata/SessionEntries?$filter=SessionId%20eq%20%27ss-1%27%20and%20Sequence%20ge%2040&$orderby=Sequence%20asc&$top=200" ); } + #[test] + fn virtual_first_turn_leaf_skips_session_entry_lookup() { + assert!(is_virtual_first_turn_leaf( + "ss-019edb98-f84e-7e11-b5ad-f14df499fa8f", + "u-ss-019edb98-f84e-7e11-b5ad-f14df499fa8f-0" + )); + assert!(!is_virtual_first_turn_leaf( + "ss-019edb98-f84e-7e11-b5ad-f14df499fa8f", + "a-2" + )); + } + #[test] fn next_session_entry_id_advances_numeric_suffix() { assert_eq!(next_session_entry_id("a", "u-1"), ("a-2".to_string(), 2));