From 5e905b11264ff1c2396cb660604c9de5a1ba0d74 Mon Sep 17 00:00:00 2001 From: Mayne Date: Sat, 1 Aug 2026 15:59:43 +0800 Subject: [PATCH 1/4] feat(remote): add resumable multipart uploads Advertise request limits and split oversized immutable objects into resumable parts backed by durable R2 multipart state. Skip oversized aggregate requests so large pushes stay below edge request limits while preserving immutable publication semantics. --- crates/graft/src/remote.rs | 481 +++++++++++++++++- .../docs/docs/reference/remote-protocol.mdx | 63 ++- .../zh/docs/reference/remote-protocol.mdx | 57 ++- .../graft-remote-cloudflare/src/backend.ts | 187 +++++++ .../graft-remote-cloudflare/src/repository.ts | 177 +++++++ packages/graft-remote/README.md | 9 + packages/graft-remote/src/handler.ts | 289 ++++++++++- packages/graft-remote/src/index.ts | 6 + packages/graft-remote/src/protocol.ts | 3 + packages/graft-remote/src/types.ts | 38 ++ packages/graft-remote/test/remote.test.ts | 144 +++++- .../test/worker.test.ts | 48 ++ 12 files changed, 1477 insertions(+), 25 deletions(-) diff --git a/crates/graft/src/remote.rs b/crates/graft/src/remote.rs index f3696332..b4eab098 100644 --- a/crates/graft/src/remote.rs +++ b/crates/graft/src/remote.rs @@ -42,10 +42,16 @@ const RECEIVE_PACK_HEADER_PACK_BYTES: &str = "x-graft-pack-bytes"; const RECEIVE_PACK_HEADER_INDEX_BYTES: &str = "x-graft-index-bytes"; const RECEIVE_PACK_HEADER_REPLACEMENT_HEX: &str = "x-graft-ref-replacement-hex"; const RECEIVE_BUNDLE_HEADER_MANIFEST_BYTES: &str = "x-graft-bundle-manifest-bytes"; +const MULTIPART_HEADER_OBJECT_BYTES: &str = "x-graft-object-bytes"; +const MULTIPART_HEADER_UPLOAD_ID: &str = "x-graft-upload-id"; +const MULTIPART_HEADER_PART_NUMBER: &str = "x-graft-part-number"; const MAX_UPLOAD_BUNDLE_MANIFEST_BYTES: usize = 64 * 1024; const MAX_UPLOAD_BUNDLE_OBJECTS: usize = 65_536; const MAX_UPLOAD_BUNDLE_PATH_BYTES: usize = 768; const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MULTIPART_DISCOVERY_THRESHOLD_BYTES: usize = 64 * 1024 * 1024; +const MAX_MULTIPART_PARTS: usize = 10_000; +const MULTIPART_PART_ATTEMPTS: usize = 3; static HTTP_REQUEST_SEQUENCE: AtomicU64 = AtomicU64::new(1); enum RemotePath<'a> { @@ -513,9 +519,43 @@ struct HttpRemote { client: reqwest::Client, probe_client: reqwest::Client, upload_client: reqwest::Client, + descriptor: Arc>, request_timeout: Duration, url: String, token: Option, + #[cfg(test)] + multipart_discovery_threshold: usize, +} + +#[derive(Debug, Deserialize)] +struct HttpRemoteDescriptor { + protocol: String, + version: u8, + #[serde(default)] + capabilities: HashSet, + #[serde(default)] + limits: HttpRemoteLimits, +} + +#[derive(Debug, Default, Deserialize)] +struct HttpRemoteLimits { + max_request_bytes: Option, + multipart_part_bytes: Option, +} + +#[derive(Debug, Deserialize)] +struct HttpMultipartStartResponse { + upload_id: String, + total_bytes: usize, + part_bytes: usize, + #[serde(default)] + uploaded_parts: Vec, +} + +#[derive(Debug, Deserialize)] +struct HttpMultipartPart { + part_number: usize, + bytes: usize, } #[derive(Debug, Deserialize)] @@ -1100,14 +1140,22 @@ impl Remote { async fn put_bundle_objects(&self, objects: &[RemoteBundleObject]) -> Result<()> { for object in objects { - let bytes = object.bytes(); - match self - .put_raw_if_not_exists(&object.path, bytes.clone()) - .await - { + let upload = match &self.backend { + RemoteBackend::Http(remote) => { + remote + .put_raw_if_not_exists_stream(&object.path, object.chunks.clone()) + .await + } + RemoteBackend::ObjectStore(_) => { + self.put_raw_if_not_exists(&object.path, object.bytes()) + .await + } + }; + match upload { Ok(()) => {} Err(err) if err.precondition_failed() && object.allow_existing => {} Err(err) if err.precondition_failed() => { + let bytes = object.bytes(); let existing = self.get_raw(&object.path).await?; if existing.as_ref() != Some(&bytes) { return Err(RemoteErr::HttpStatus { @@ -1287,9 +1335,12 @@ impl HttpRemote { client, probe_client, upload_client, + descriptor: Arc::new(tokio::sync::OnceCell::new()), request_timeout, url: url.trim_end_matches('/').to_string(), token, + #[cfg(test)] + multipart_discovery_threshold: MULTIPART_DISCOVERY_THRESHOLD_BYTES, } } @@ -1310,6 +1361,115 @@ impl HttpRemote { url } + async fn descriptor(&self) -> Result<&HttpRemoteDescriptor> { + self.descriptor + .get_or_try_init(|| async { + let response = self + .send( + self.probe_request(reqwest::Method::GET, self.url.clone()), + "descriptor", + Some(0), + ) + .await?; + let response = Self::check_response(response, &self.url).await?; + let bytes = response + .bytes() + .await + .map_err(|err| RemoteErr::http_transport("descriptor_body", err))?; + let descriptor: HttpRemoteDescriptor = + serde_json::from_slice(&bytes).map_err(|err| RemoteErr::HttpStatus { + status: 502, + path: self.url.clone(), + message: format!("invalid remote descriptor JSON: {err}"), + })?; + if descriptor.protocol != "graft-remote" || descriptor.version != 1 { + return Err(RemoteErr::HttpStatus { + status: 502, + path: self.url.clone(), + message: "remote descriptor identifies an unsupported protocol".to_string(), + }); + } + if descriptor + .limits + .max_request_bytes + .is_some_and(|bytes| bytes == 0) + || descriptor + .limits + .multipart_part_bytes + .is_some_and(|bytes| bytes == 0) + { + return Err(RemoteErr::HttpStatus { + status: 502, + path: self.url.clone(), + message: "remote descriptor contains invalid request limits".to_string(), + }); + } + if descriptor.capabilities.contains("multipart-object") + && descriptor.limits.multipart_part_bytes.is_none() + { + return Err(RemoteErr::HttpStatus { + status: 502, + path: self.url.clone(), + message: "multipart remote does not advertise a part size".to_string(), + }); + } + Ok(descriptor) + }) + .await + } + + async fn multipart_part_bytes(&self, content_length: usize) -> Result> { + let descriptor = match self.descriptor.get() { + Some(descriptor) => descriptor, + None if content_length <= self.multipart_discovery_threshold() => return Ok(None), + None => self.descriptor().await?, + }; + if !descriptor.capabilities.contains("multipart-object") { + return Ok(None); + } + let direct_limit = descriptor + .limits + .max_request_bytes + .unwrap_or_else(|| self.multipart_discovery_threshold()); + if content_length <= direct_limit { + return Ok(None); + } + let part_bytes = descriptor.limits.multipart_part_bytes.unwrap(); + if content_length.div_ceil(part_bytes) > MAX_MULTIPART_PARTS { + return Err(RemoteErr::HttpStatus { + status: 413, + path: self.url.clone(), + message: "immutable object exceeds the advertised multipart limit".to_string(), + }); + } + Ok(Some(part_bytes)) + } + + async fn aggregate_request_requires_multipart(&self, content_length: usize) -> Result { + if content_length <= self.multipart_discovery_threshold() { + return Ok(false); + } + let descriptor = self.descriptor().await?; + if !descriptor.capabilities.contains("multipart-object") { + return Ok(false); + } + Ok(descriptor + .limits + .max_request_bytes + .is_some_and(|limit| content_length > limit)) + } + + fn multipart_discovery_threshold(&self) -> usize { + #[cfg(test)] + { + self.multipart_discovery_threshold + } + #[cfg(not(test))] + { + MULTIPART_DISCOVERY_THRESHOLD_BYTES + } + } + fn request(&self, method: reqwest::Method, url: String) -> reqwest::RequestBuilder { self.request_with(&self.client, method, url) .timeout(self.request_timeout) @@ -1692,20 +1852,7 @@ impl HttpRemote { } async fn put_raw_if_not_exists(&self, path: &str, bytes: Bytes) -> Result<()> { - let request_bytes = bytes.len() as u64; - let response = self - .send( - self.upload_request( - reqwest::Method::PUT, - self.raw_url("raw-if-not-exists", path), - ) - .body(bytes), - "immutable_put", - Some(request_bytes), - ) - .await?; - Self::check_response(response, path).await?; - Ok(()) + self.put_raw_if_not_exists_chunks(path, vec![bytes]).await } async fn put_raw_if_not_exists_stream>( @@ -1713,7 +1860,11 @@ impl HttpRemote { path: &str, chunks: I, ) -> Result<()> { - let chunks = chunks.into_iter().collect::>(); + self.put_raw_if_not_exists_chunks(path, chunks.into_iter().collect()) + .await + } + + async fn put_raw_if_not_exists_chunks(&self, path: &str, chunks: Vec) -> Result<()> { let content_length = chunks.iter().try_fold(0_usize, |total, chunk| { total .checked_add(chunk.len()) @@ -1723,6 +1874,12 @@ impl HttpRemote { message: "streamed upload length exceeds usize".to_string(), }) })?; + if let Some(part_bytes) = self.multipart_part_bytes(content_length).await? { + return self + .put_raw_if_not_exists_multipart(path, &chunks, content_length, part_bytes) + .await; + } + let body = reqwest::Body::wrap_stream(stream::iter( chunks.into_iter().map(Ok::), )); @@ -1742,6 +1899,133 @@ impl HttpRemote { Ok(()) } + async fn put_raw_if_not_exists_multipart( + &self, + path: &str, + chunks: &[Bytes], + content_length: usize, + part_bytes: usize, + ) -> Result<()> { + let response = self + .send( + self.upload_request(reqwest::Method::POST, self.raw_url("multipart-start", path)) + .header(reqwest::header::CONTENT_LENGTH, 0) + .header(MULTIPART_HEADER_OBJECT_BYTES, content_length), + "multipart_start", + Some(0), + ) + .await?; + let response = Self::check_response(response, path).await?; + let response_bytes = response + .bytes() + .await + .map_err(|err| RemoteErr::http_transport("multipart_start_body", err))?; + let upload: HttpMultipartStartResponse = + serde_json::from_slice(&response_bytes).map_err(|err| RemoteErr::HttpStatus { + status: 502, + path: path.to_string(), + message: format!("invalid multipart-start response: {err}"), + })?; + validate_multipart_start(&upload, content_length, part_bytes, path)?; + let uploaded_parts = upload + .uploaded_parts + .iter() + .map(|part| (part.part_number, part.bytes)) + .collect::>(); + let parts = multipart_chunks(chunks, part_bytes); + for (index, part) in parts.iter().enumerate() { + let part_number = index + 1; + let length = part.iter().map(Bytes::len).sum::(); + if uploaded_parts.get(&part_number) == Some(&length) { + continue; + } + self.put_multipart_part(path, &upload.upload_id, part_number, part, length) + .await?; + } + + let completion = self + .send( + self.upload_request( + reqwest::Method::POST, + self.raw_url("multipart-complete", path), + ) + .header(reqwest::header::CONTENT_LENGTH, 0) + .header(MULTIPART_HEADER_UPLOAD_ID, &upload.upload_id), + "multipart_complete", + Some(0), + ) + .await; + let response = match completion { + Ok(response) => response, + Err(error) => { + return match self.has_raw(path).await { + Ok(true) => Ok(()), + _ => Err(error), + }; + } + }; + if response.status().is_success() { + Self::check_response(response, path).await?; + return Ok(()); + } + let status = response.status().as_u16(); + let error = Self::check_response(response, path).await.unwrap_err(); + if matches!(status, 500..=599) && self.has_raw(path).await.unwrap_or(false) { + return Ok(()); + } + Err(error) + } + + async fn put_multipart_part( + &self, + path: &str, + upload_id: &str, + part_number: usize, + chunks: &[Bytes], + content_length: usize, + ) -> Result<()> { + let mut last_error = None; + for attempt in 0..MULTIPART_PART_ATTEMPTS { + let request_chunks = chunks.to_vec(); + let body = reqwest::Body::wrap_stream(stream::iter( + request_chunks.into_iter().map(Ok::), + )); + let response = self + .send( + self.upload_request(reqwest::Method::PUT, self.raw_url("multipart-part", path)) + .header(reqwest::header::CONTENT_LENGTH, content_length) + .header(MULTIPART_HEADER_UPLOAD_ID, upload_id) + .header(MULTIPART_HEADER_PART_NUMBER, part_number) + .body(body), + "multipart_part", + Some(content_length as u64), + ) + .await; + match response { + Ok(response) if response.status().is_success() => { + Self::check_response(response, path).await?; + return Ok(()); + } + Ok(response) + if attempt + 1 < MULTIPART_PART_ATTEMPTS + && matches!(response.status().as_u16(), 429 | 500..=599) => + { + last_error = Some(Self::check_response(response, path).await.unwrap_err()); + } + Ok(response) => { + Self::check_response(response, path).await?; + unreachable!("non-success multipart response passed validation") + } + Err(error) if attempt + 1 < MULTIPART_PART_ATTEMPTS => { + last_error = Some(error); + } + Err(error) => return Err(error), + } + tokio::time::sleep(Duration::from_millis(100 * (attempt as u64 + 1))).await; + } + Err(last_error.unwrap()) + } + async fn receive_pack( &self, pack: &RemoteObjectPack, @@ -1758,6 +2042,12 @@ impl HttpRemote { path: ref_path.to_string(), message: "receive-pack body length exceeds usize".to_string(), })?; + if self + .aggregate_request_requires_multipart(content_length) + .await? + { + return Ok(HttpReceivePackResult::RetryIndividually); + } let body = reqwest::Body::wrap_stream(stream::iter( [pack.pack.clone(), pack.index.clone()] .into_iter() @@ -1793,6 +2083,10 @@ impl HttpRemote { Self::drain_response(response).await?; return Ok(HttpReceivePackResult::Unsupported); } + if response.status().as_u16() == 413 { + Self::drain_response(response).await?; + return Ok(HttpReceivePackResult::RetryIndividually); + } Self::check_response(response, ref_path).await?; Ok(HttpReceivePackResult::Published) } @@ -1846,6 +2140,12 @@ impl HttpRemote { path: ref_path.to_string(), message: "receive-bundle body length exceeds usize".to_string(), })?; + if self + .aggregate_request_requires_multipart(content_length) + .await? + { + return Ok(HttpReceivePackResult::RetryIndividually); + } let chunks = std::iter::once(Bytes::from(manifest.clone())) .chain( objects @@ -2220,6 +2520,75 @@ fn safe_server_timing_name(value: &str) -> Option<&'static str> { } } +fn validate_multipart_start( + upload: &HttpMultipartStartResponse, + total_bytes: usize, + part_bytes: usize, + path: &str, +) -> Result<()> { + if upload.upload_id.is_empty() + || upload.upload_id.len() > 1_024 + || upload.upload_id.chars().any(char::is_control) + || upload.total_bytes != total_bytes + || upload.part_bytes != part_bytes + { + return Err(RemoteErr::HttpStatus { + status: 502, + path: path.to_string(), + message: "multipart-start returned an invalid upload session".to_string(), + }); + } + let part_count = total_bytes.div_ceil(part_bytes); + let mut previous = 0; + for part in &upload.uploaded_parts { + if part.part_number <= previous || part.part_number > part_count { + return Err(RemoteErr::HttpStatus { + status: 502, + path: path.to_string(), + message: "multipart-start returned invalid uploaded parts".to_string(), + }); + } + let expected = if part.part_number == part_count { + total_bytes - part_bytes * (part_count - 1) + } else { + part_bytes + }; + if part.bytes != expected { + return Err(RemoteErr::HttpStatus { + status: 502, + path: path.to_string(), + message: "multipart-start returned an invalid uploaded part size".to_string(), + }); + } + previous = part.part_number; + } + Ok(()) +} + +fn multipart_chunks(chunks: &[Bytes], part_bytes: usize) -> Vec> { + let total_bytes = chunks.iter().map(Bytes::len).sum::(); + let mut parts = Vec::with_capacity(total_bytes.div_ceil(part_bytes)); + let mut part = Vec::new(); + let mut part_len = 0; + for chunk in chunks { + let mut offset = 0; + while offset < chunk.len() { + let take = (part_bytes - part_len).min(chunk.len() - offset); + part.push(chunk.slice(offset..offset + take)); + part_len += take; + offset += take; + if part_len == part_bytes { + parts.push(std::mem::take(&mut part)); + part_len = 0; + } + } + } + if part_len != 0 { + parts.push(part); + } + parts +} + fn percent_encode_path(path: &str) -> String { path.split('/') .map(percent_encode_component) @@ -3691,6 +4060,78 @@ mod tests { ); } + #[tokio::test] + async fn large_http_upload_uses_and_resumes_multipart_parts() { + let descriptor = serde_json::json!({ + "protocol": "graft-remote", + "version": 1, + "repository": "org/repo", + "capabilities": ["multipart-object"], + "limits": { + "max_request_bytes": 64 * 1024, + "multipart_part_bytes": 32 * 1024, + }, + }) + .to_string(); + let start = serde_json::json!({ + "upload_id": "upload-1", + "total_bytes": 70 * 1024, + "part_bytes": 32 * 1024, + "uploaded_parts": [{ "part_number": 1, "bytes": 32 * 1024 }], + }) + .to_string(); + let json_response = |body: &str| { + format!( + "HTTP/1.1 200 OK\r\nGraft-Protocol: 1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + }; + let responses = [ + json_response(&descriptor), + json_response(&start), + "HTTP/1.1 204 No Content\r\nGraft-Protocol: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + "HTTP/1.1 204 No Content\r\nGraft-Protocol: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + "HTTP/1.1 204 No Content\r\nGraft-Protocol: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + ]; + let response_refs = responses.iter().map(String::as_str).collect::>(); + let (url, requests) = serve_http_messages(&response_refs).await; + let mut remote = HttpRemote::new(url, None).unwrap(); + remote.multipart_discovery_threshold = 64 * 1024; + let payload = Bytes::from(vec![7_u8; 70 * 1024]); + + remote + .put_raw_if_not_exists_stream("segments/large", [payload.clone()]) + .await + .unwrap(); + + let requests = requests.await.unwrap(); + assert_eq!(requests.len(), 5); + assert!(String::from_utf8_lossy(&requests[0]).starts_with("GET /org/repo HTTP/1.1")); + assert!( + String::from_utf8_lossy(&requests[1]) + .starts_with("POST /org/repo/multipart-start/segments/large") + ); + assert!( + String::from_utf8_lossy(&requests[2]) + .lines() + .any(|line| line.eq_ignore_ascii_case("x-graft-part-number: 2")) + ); + assert_eq!( + http_request_body(&requests[2]), + &payload[32 * 1024..64 * 1024] + ); + assert!( + String::from_utf8_lossy(&requests[3]) + .lines() + .any(|line| line.eq_ignore_ascii_case("x-graft-part-number: 3")) + ); + assert_eq!(http_request_body(&requests[3]), &payload[64 * 1024..]); + assert!( + String::from_utf8_lossy(&requests[4]) + .starts_with("POST /org/repo/multipart-complete/segments/large") + ); + } + #[tokio::test] async fn streamed_http_upload_completes_when_remote_rejects_before_reading_body() { let (url, request) = serve_http_response("412 Precondition Failed", &["1"]).await; diff --git a/docs/src/content/docs/docs/reference/remote-protocol.mdx b/docs/src/content/docs/docs/reference/remote-protocol.mdx index 8aa6ff42..45f6e9f8 100644 --- a/docs/src/content/docs/docs/reference/remote-protocol.mdx +++ b/docs/src/content/docs/docs/reference/remote-protocol.mdx @@ -93,9 +93,14 @@ issuance, tenancy, and ACL storage are service concerns outside this protocol. "upload-bundle", "receive-pack", "receive-bundle", + "multipart-object", "cas", "cad" - ] + ], + "limits": { + "max_request_bytes": 67108864, + "multipart_part_bytes": 16777216 + } } ``` @@ -142,6 +147,10 @@ All paths below are relative to `{base}`. | `POST /upload-bundle/` | Stream a ref snapshot and its immutable storage. | `200 OK` | | `POST /receive-pack/` | Publish one object pack and atomically update a ref. | `204 No Content` | | `POST /receive-bundle/` | Publish immutable objects, a pack, and a ref. | `204 No Content` | +| `POST /multipart-start/` | Start or resume a multipart immutable upload. | `200 OK` | +| `PUT /multipart-part/` | Upload one numbered part. | `204 No Content` | +| `POST /multipart-complete/` | Assemble all uploaded parts at the immutable key. | `204 No Content` | +| `DELETE /multipart-abort/` | Abort an incomplete multipart upload. | `204 No Content` | | `POST /cas/` | Atomically compare and replace metadata. | `204 No Content` | | `POST /cad/` | Atomically compare and delete metadata. | `204 No Content` | | `GET /list?prefix=` | Recursively list matching keys. | `200 OK` | @@ -285,6 +294,58 @@ verify the collision before retrying publication. Clients MUST fall back to individual immutable writes followed by `receive-pack` or `/cas` when `receive-bundle` returns `404` or `405`. +## Multipart Immutable Objects + +`multipart-object` is an optional version 1 capability for immutable objects +that exceed an HTTP gateway's request-body limit. It changes only the transfer: +the completed object remains at the original key and retains the same logical +content identifier. + +The descriptor MUST include `limits.multipart_part_bytes` when this capability +is advertised. A service SHOULD also publish `limits.max_request_bytes` so a +client can skip an aggregate `receive-bundle` or `receive-pack` request that +cannot reach the application. A client starts or resumes an upload with: + +```http +POST /multipart-start/segments/ +Content-Length: 0 +x-graft-object-bytes: +``` + +The response identifies the durable session and the parts already stored: + +```json +{ + "upload_id": "opaque-upload-id", + "total_bytes": 111249535, + "part_bytes": 16777216, + "uploaded_parts": [{ "part_number": 1, "bytes": 16777216 }] +} +``` + +Clients MUST send all missing parts in ascending order. Part numbers start at +one. Every non-final part has exactly the advertised part size; the final part +contains the remainder. + +```http +PUT /multipart-part/segments/ +Content-Length: +x-graft-upload-id: +x-graft-part-number: +``` + +Re-uploading a part number replaces that part in the same session, which makes +a lost response safe to retry. Repeating `multipart-start` for the same key and +length returns the same session and completed-part list. After all parts are +present, `POST /multipart-complete/` with `x-graft-upload-id` and an empty +body atomically exposes the complete immutable object. `DELETE +/multipart-abort/` with the same header releases an incomplete session. + +The start and complete operations return `412` when the immutable target +already exists. Clients apply the same collision policy used by +`raw-if-not-exists`. Multipart completion never publishes a ref; the client +still performs `receive-pack` or CAS only after all immutable objects exist. + ## Compare Operations `POST /cas` and `POST /cad` carry the expected value in two headers: diff --git a/docs/src/content/docs/zh/docs/reference/remote-protocol.mdx b/docs/src/content/docs/zh/docs/reference/remote-protocol.mdx index 71d006f0..6a2213d3 100644 --- a/docs/src/content/docs/zh/docs/reference/remote-protocol.mdx +++ b/docs/src/content/docs/zh/docs/reference/remote-protocol.mdx @@ -79,9 +79,14 @@ ACL 存储不属于本协议。 "upload-bundle", "receive-pack", "receive-bundle", + "multipart-object", "cas", "cad" - ] + ], + "limits": { + "max_request_bytes": 67108864, + "multipart_part_bytes": 16777216 + } } ``` @@ -124,6 +129,10 @@ encoding 必须被拒绝。服务可以公布 key/body 限制，并在超限时 | `POST /upload-bundle/` | 流式返回 ref 快照及其不可变存储。 | `200 OK` | | `POST /receive-pack/` | 发布一个 object pack 并原子更新 ref。 | `204 No Content` | | `POST /receive-bundle/` | 发布不可变对象、pack 并原子更新 ref。 | `204 No Content` | +| `POST /multipart-start/` | 开始或恢复不可变对象的分片上传。 | `200 OK` | +| `PUT /multipart-part/` | 上传一个编号分片。 | `204 No Content` | +| `POST /multipart-complete/` | 将全部分片组装到原不可变 key。 | `204 No Content` | +| `DELETE /multipart-abort/` | 终止未完成的分片上传。 | `204 No Content` | | `POST /cas/` | 原子 compare-and-swap。 | `204 No Content` | | `POST /cad/` | 原子 compare-and-delete。 | `204 No Content` | | `GET /list?prefix=` | 递归列举 key。 | `200 OK` | @@ -245,6 +254,52 @@ manifest、全部对象、pack 和 index 的长度之和。服务必须按 manif 通过独立的 version 1 路径读取并校验冲突后再重试发布。`receive-bundle` 返回 `404` 或 `405` 时，client 必须回退到逐个不可变对象写入，再调用 `receive-pack` 或 `/cas`。 +## 不可变对象分片上传 + +`multipart-object` 是 version 1 的可选 capability，用于超过 HTTP 网关 request body +限制的不可变对象。它只改变传输方式：完成后的对象仍位于原 key，并保持相同的逻辑 +content ID。 + +公布该 capability 时，descriptor 必须包含 `limits.multipart_part_bytes`；服务也应该 +公布 `limits.max_request_bytes`，让 client 在请求不可能到达应用时直接跳过聚合的 +`receive-bundle` 或 `receive-pack`。开始或恢复上传： + +```http +POST /multipart-start/segments/ +Content-Length: 0 +x-graft-object-bytes: <总字节数> +``` + +响应返回持久 upload session 与已经完成的 part： + +```json +{ + "upload_id": "opaque-upload-id", + "total_bytes": 111249535, + "part_bytes": 16777216, + "uploaded_parts": [{ "part_number": 1, "bytes": 16777216 }] +} +``` + +part 从 1 开始编号。除最后一个 part 外，其余 part 必须等于 descriptor 公布的大小； +最后一个 part 携带余数： + +```http +PUT /multipart-part/segments/ +Content-Length: +x-graft-upload-id: +x-graft-part-number: <正十进制编号> +``` + +同一 session 重传相同编号会替换该 part，因此响应丢失后可以安全重试。对相同 key 和 +长度再次调用 `multipart-start`，必须返回同一 session 和已完成 part 列表。全部 part +存在后，使用空 body 和 `x-graft-upload-id` 调用 `POST /multipart-complete/`， +把完整对象暴露在原不可变 key；`DELETE /multipart-abort/` 释放未完成 session。 + +目标已存在时，start/complete 返回 `412`，client 沿用 `raw-if-not-exists` 的 collision +策略。multipart complete 不发布 ref；所有不可变对象完成后，client 仍需执行 +`receive-pack` 或 CAS。 + ## Compare 操作 `POST /cas` 和 `POST /cad` 用两个 header 携带 expected value： diff --git a/packages/graft-remote-cloudflare/src/backend.ts b/packages/graft-remote-cloudflare/src/backend.ts index f4c7a21e..6067ea6a 100644 --- a/packages/graft-remote-cloudflare/src/backend.ts +++ b/packages/graft-remote-cloudflare/src/backend.ts @@ -4,6 +4,8 @@ import { type GraftByteRange, type GraftListQuery, type GraftListResult, + type GraftMultipartBackend, + type GraftMultipartUpload, type GraftObject, type GraftObjectMetadata, type GraftRepositoryBackend, @@ -13,10 +15,22 @@ import { import type { RepositoryDurableObject } from "./repository"; +const R2_MIN_MULTIPART_PART_BYTES = 5 * 1024 * 1024; +const R2_MAX_MULTIPART_PART_BYTES = 5 * 1024 * 1024 * 1024; +const R2_MAX_MULTIPART_PARTS = 10_000; + export class CloudflareRepositoryBackend implements GraftRepositoryBackend { readonly #objects: R2Bucket; readonly #repositoryId: string; readonly #metadata: DurableObjectStub; + readonly multipart: GraftMultipartBackend = { + start: async (path, totalBytes, partBytes) => + await this.startMultipart(path, totalBytes, partBytes), + uploadPart: async (path, uploadId, partNumber, value, contentLength) => + await this.uploadMultipartPart(path, uploadId, partNumber, value, contentLength), + complete: async (path, uploadId) => await this.completeMultipart(path, uploadId), + abort: async (path, uploadId) => await this.abortMultipart(path, uploadId), + }; constructor(storage: CloudflareRepositoryStorage, repositoryId: string) { this.#objects = storage.objects; @@ -128,6 +142,152 @@ export class CloudflareRepositoryBackend implements GraftRepositoryBackend { return `repositories/${this.#repositoryId}/objects/${path}`; } + private async startMultipart( + path: string, + totalBytes: number, + partBytes: number, + ): Promise { + validateR2MultipartShape(totalBytes, partBytes); + const key = this.r2Key(path); + if ((await this.#objects.head(key)) !== null) return null; + + const existing = await this.#metadata.getMultipartUpload(path); + if (existing !== null) { + if (existing.totalBytes === totalBytes && existing.partBytes === partBytes) { + return publicMultipartUpload(existing); + } + await this.discardMultipart(path, existing.uploadId, key); + } + + const upload = await this.#objects.createMultipartUpload(key, { + httpMetadata: { contentType: "application/octet-stream" }, + }); + let retained = false; + try { + retained = await this.#metadata.createMultipartUpload( + path, + upload.uploadId, + totalBytes, + partBytes, + ); + if (retained) { + return { + uploadId: upload.uploadId, + totalBytes, + partBytes, + uploadedParts: [], + }; + } + } finally { + if (!retained) { + try { + await upload.abort(); + } catch { + // Another request won the durable session. R2 also expires abandoned uploads. + } + } + } + + const winner = await this.#metadata.getMultipartUpload(path); + if ( + winner === null || + winner.totalBytes !== totalBytes || + winner.partBytes !== partBytes + ) { + throw new Error("Multipart upload session changed while it was created"); + } + return publicMultipartUpload(winner); + } + + private async uploadMultipartPart( + path: string, + uploadId: string, + partNumber: number, + value: ReadableStream, + contentLength: number, + ): Promise { + const state = await this.requireMultipart(path, uploadId); + const partCount = Math.ceil(state.totalBytes / state.partBytes); + const expectedBytes = + partNumber === partCount + ? state.totalBytes - state.partBytes * (partCount - 1) + : state.partBytes; + if (partNumber < 1 || partNumber > partCount || contentLength !== expectedBytes) { + throw new RangeError("Multipart part does not match the upload session"); + } + + const fixed = fixedR2Body(value, contentLength); + try { + const uploaded = await this.#objects + .resumeMultipartUpload(this.r2Key(path), uploadId) + .uploadPart(partNumber, fixed.body); + await fixed.finish(true); + await this.#metadata.recordMultipartPart( + path, + uploadId, + uploaded.partNumber, + uploaded.etag, + contentLength, + ); + } catch (error) { + await fixed.cancel(); + await fixed.finish(false); + throw error; + } + } + + private async completeMultipart(path: string, uploadId: string): Promise { + const key = this.r2Key(path); + if ((await this.#objects.head(key)) !== null) { + await this.discardMultipart(path, uploadId, key); + return false; + } + const state = await this.requireMultipart(path, uploadId); + const partCount = Math.ceil(state.totalBytes / state.partBytes); + if ( + state.uploadedParts.length !== partCount || + state.uploadedParts.some((part, index) => part.partNumber !== index + 1) + ) { + throw new RangeError("Multipart upload is incomplete"); + } + const object = await this.#objects + .resumeMultipartUpload(key, uploadId) + .complete( + state.uploadedParts.map((part) => ({ + partNumber: part.partNumber, + etag: part.etag, + })), + ); + if (object.size !== state.totalBytes) { + throw new Error("Completed multipart object has an unexpected size"); + } + await this.#metadata.deleteMultipartUpload(path, uploadId); + return true; + } + + private async abortMultipart(path: string, uploadId: string): Promise { + const state = await this.#metadata.getMultipartUpload(path); + if (state === null || state.uploadId !== uploadId) return; + await this.discardMultipart(path, uploadId, this.r2Key(path)); + } + + private async requireMultipart(path: string, uploadId: string) { + const state = await this.#metadata.getMultipartUpload(path); + if (state === null || state.uploadId !== uploadId) { + throw new RangeError("Multipart upload session does not exist"); + } + return state; + } + + private async discardMultipart(path: string, uploadId: string, key: string): Promise { + try { + await this.#objects.resumeMultipartUpload(key, uploadId).abort(); + } catch { + // The upload may already be complete or expired. The durable session is stale either way. + } + await this.#metadata.deleteMultipartUpload(path, uploadId); + } + private async putImmutable( path: string, value: GraftWriteBody, @@ -150,6 +310,33 @@ export class CloudflareRepositoryBackend implements GraftRepositoryBackend { } } +function publicMultipartUpload( + state: Awaited> & {}, +): GraftMultipartUpload { + return { + uploadId: state.uploadId, + totalBytes: state.totalBytes, + partBytes: state.partBytes, + uploadedParts: state.uploadedParts.map((part) => ({ + partNumber: part.partNumber, + bytes: part.bytes, + })), + }; +} + +function validateR2MultipartShape(totalBytes: number, partBytes: number): void { + if ( + !Number.isSafeInteger(totalBytes) || + totalBytes < 1 || + !Number.isSafeInteger(partBytes) || + partBytes < R2_MIN_MULTIPART_PART_BYTES || + partBytes > R2_MAX_MULTIPART_PART_BYTES || + Math.ceil(totalBytes / partBytes) > R2_MAX_MULTIPART_PARTS + ) { + throw new RangeError("Object is outside the supported R2 multipart limits"); + } +} + export interface CloudflareRepositoryStorage { objects: R2Bucket; repositories: DurableObjectNamespace; diff --git a/packages/graft-remote-cloudflare/src/repository.ts b/packages/graft-remote-cloudflare/src/repository.ts index b67e8148..b7d3ff64 100644 --- a/packages/graft-remote-cloudflare/src/repository.ts +++ b/packages/graft-remote-cloudflare/src/repository.ts @@ -15,11 +15,32 @@ interface ChangeRow { changed: number; } +interface MultipartUploadRow { + [key: string]: SqlStorageValue; + upload_id: string; + total_bytes: number; + part_bytes: number; +} + +interface MultipartPartRow { + [key: string]: SqlStorageValue; + part_number: number; + etag: string; + bytes: number; +} + export interface MetadataListResult { paths: string[]; hasMore: boolean; } +export interface MultipartUploadState { + uploadId: string; + totalBytes: number; + partBytes: number; + uploadedParts: Array<{ partNumber: number; etag: string; bytes: number }>; +} + export class RepositoryDurableObject extends DurableObject { constructor(ctx: DurableObjectState, env: Cloudflare.Env) { super(ctx, env); @@ -29,6 +50,26 @@ export class RepositoryDurableObject extends DurableObject { value BLOB NOT NULL ) `); + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS multipart_uploads ( + path TEXT PRIMARY KEY, + upload_id TEXT NOT NULL UNIQUE, + total_bytes INTEGER NOT NULL CHECK (total_bytes > 0), + part_bytes INTEGER NOT NULL CHECK (part_bytes > 0), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `); + this.ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS multipart_parts ( + path TEXT NOT NULL, + upload_id TEXT NOT NULL, + part_number INTEGER NOT NULL CHECK (part_number > 0), + etag TEXT NOT NULL, + bytes INTEGER NOT NULL CHECK (bytes > 0), + PRIMARY KEY (path, part_number) + ) + `); } async headMetadata(path: string): Promise { @@ -131,6 +172,121 @@ export class RepositoryDurableObject extends DurableObject { return { paths: rows.slice(0, limit), hasMore: rows.length > limit }; } + async getMultipartUpload(path: string): Promise { + const upload = this.ctx.storage.sql + .exec( + "SELECT upload_id, total_bytes, part_bytes FROM multipart_uploads WHERE path = ?", + path, + ) + .toArray()[0]; + if (upload === undefined) return null; + const uploadedParts = this.ctx.storage.sql + .exec( + `SELECT part_number, etag, bytes FROM multipart_parts + WHERE path = ? AND upload_id = ? ORDER BY part_number`, + path, + upload.upload_id, + ) + .toArray() + .map((part) => ({ + partNumber: part.part_number, + etag: part.etag, + bytes: part.bytes, + })); + return { + uploadId: upload.upload_id, + totalBytes: upload.total_bytes, + partBytes: upload.part_bytes, + uploadedParts, + }; + } + + async createMultipartUpload( + path: string, + uploadId: string, + totalBytes: number, + partBytes: number, + ): Promise { + validateMultipartIdentity(path, uploadId, totalBytes, partBytes); + const now = Date.now(); + return ( + this.ctx.storage.sql + .exec( + `INSERT OR IGNORE INTO multipart_uploads( + path, upload_id, total_bytes, part_bytes, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?) RETURNING 1 AS changed`, + path, + uploadId, + totalBytes, + partBytes, + now, + now, + ) + .toArray().length === 1 + ); + } + + async recordMultipartPart( + path: string, + uploadId: string, + partNumber: number, + etag: string, + bytes: number, + ): Promise { + const upload = await this.getMultipartUpload(path); + if (upload === null || upload.uploadId !== uploadId) { + throw new RangeError("Multipart upload session does not exist"); + } + const partCount = Math.ceil(upload.totalBytes / upload.partBytes); + const expectedBytes = + partNumber === partCount + ? upload.totalBytes - upload.partBytes * (partCount - 1) + : upload.partBytes; + if ( + !Number.isSafeInteger(partNumber) || + partNumber < 1 || + partNumber > partCount || + bytes !== expectedBytes || + etag.length < 1 || + etag.length > 1_024 || + /[\u0000-\u001f\u007f]/.test(etag) + ) { + throw new RangeError("Invalid multipart part metadata"); + } + this.ctx.storage.sql.exec( + `INSERT INTO multipart_parts(path, upload_id, part_number, etag, bytes) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(path, part_number) DO UPDATE SET + upload_id = excluded.upload_id, + etag = excluded.etag, + bytes = excluded.bytes`, + path, + uploadId, + partNumber, + etag, + bytes, + ); + this.ctx.storage.sql.exec( + "UPDATE multipart_uploads SET updated_at = ? WHERE path = ? AND upload_id = ?", + Date.now(), + path, + uploadId, + ); + } + + async deleteMultipartUpload(path: string, uploadId: string): Promise { + this.ctx.storage.sql.exec( + "DELETE FROM multipart_parts WHERE path = ? AND upload_id = ?", + path, + uploadId, + ); + this.ctx.storage.sql.exec( + "DELETE FROM multipart_uploads WHERE path = ? AND upload_id = ?", + path, + uploadId, + ); + } + private readMetadata(path: string): Uint8Array | undefined { const row = this.ctx.storage.sql .exec("SELECT value FROM metadata WHERE path = ?", path) @@ -139,6 +295,27 @@ export class RepositoryDurableObject extends DurableObject { } } +function validateMultipartIdentity( + path: string, + uploadId: string, + totalBytes: number, + partBytes: number, +): void { + if ( + path.length < 1 || + path.length > 2_048 || + uploadId.length < 1 || + uploadId.length > 1_024 || + /[\u0000-\u001f\u007f]/.test(path + uploadId) || + !Number.isSafeInteger(totalBytes) || + totalBytes < 1 || + !Number.isSafeInteger(partBytes) || + partBytes < 1 + ) { + throw new RangeError("Invalid multipart upload metadata"); + } +} + function exactArrayBuffer(value: Uint8Array): ArrayBuffer { return new Uint8Array(value).buffer; } diff --git a/packages/graft-remote/README.md b/packages/graft-remote/README.md index 3886c5f2..560baa16 100644 --- a/packages/graft-remote/README.md +++ b/packages/graft-remote/README.md @@ -39,6 +39,10 @@ interface User { } const handleRemote = createGraftRemoteHandler({ + limits: { + maxRequestBytes: 64 * 1024 * 1024, + multipartPartBytes: 16 * 1024 * 1024, + }, async authenticate({ request }) { return await authenticateRequest(request); }, @@ -106,6 +110,7 @@ interface GraftRepositoryBackend { expected: Uint8Array | undefined, ): MaybePromise; list(query: GraftListQuery): MaybePromise; + multipart?: GraftMultipartBackend; } ``` @@ -120,6 +125,10 @@ Backend guarantees: clone response without buffering the repository in memory. - `options.contentLength` is the exact length of a framed immutable body, such as each object in a `receive-pack` or `receive-bundle` request. +- `multipart` optionally stores one logical immutable object through resumable + parts. The protocol engine advertises it as `multipart-object`, validates + every part against `limits.multipartPartBytes`, and keeps the final object at + the original repository path. - Repository instances are isolated by `repository.id`. ## Authentication and authorization diff --git a/packages/graft-remote/src/handler.ts b/packages/graft-remote/src/handler.ts index 04396039..31765fa8 100644 --- a/packages/graft-remote/src/handler.ts +++ b/packages/graft-remote/src/handler.ts @@ -3,6 +3,9 @@ import { MAX_LIST_LIMIT, MAX_METADATA_BYTES, MAX_UPLOAD_BUNDLE_OBJECTS, + MULTIPART_HEADER_OBJECT_BYTES, + MULTIPART_HEADER_PART_NUMBER, + MULTIPART_HEADER_UPLOAD_ID, PROTOCOL_HEADER, PROTOCOL_VERSION, RECEIVE_BUNDLE_HEADER_MANIFEST_BYTES, @@ -48,18 +51,30 @@ const OPERATIONS = new Set([ "upload-bundle", "receive-pack", "receive-bundle", + "multipart-start", + "multipart-part", + "multipart-complete", + "multipart-abort", "cas", "cad", "list", ]); const UPLOAD_BUNDLE_PREFETCH_OBJECTS = 8; +const DEFAULT_MULTIPART_PART_BYTES = 16 * 1024 * 1024; +const MAX_MULTIPART_PARTS = 10_000; + +interface NormalizedRemoteLimits { + maxRequestBytes?: number; + multipartPartBytes: number; +} export function createGraftRemoteHandler( options: GraftRemoteOptions, ): GraftRemoteHandler { + const limits = normalizeRemoteLimits(options.limits); return async (request): Promise => { try { - return await handleRequest(request, options); + return await handleRequest(request, options, limits); } catch (error) { if (options.onError !== undefined) { try { @@ -83,6 +98,7 @@ export function createGraftRemoteHandler( input: GraftHandlerRequest, options: GraftRemoteOptions, + limits: NormalizedRemoteLimits, ): Promise { const principal = await options.authenticate?.(input); if (input.request.headers.get(PROTOCOL_HEADER) !== PROTOCOL_VERSION) { @@ -132,11 +148,21 @@ async function handleRequest( if (operation === "descriptor") { rejectUnexpectedQuery(url); + const capabilities: string[] = [...GRAFT_REMOTE_CAPABILITIES]; + if (backend.multipart !== undefined) capabilities.push("multipart-object"); return jsonResponse({ protocol: "graft-remote", version: 1, repository: repository.id, - capabilities: [...GRAFT_REMOTE_CAPABILITIES], + capabilities, + limits: { + ...(limits.maxRequestBytes === undefined + ? {} + : { max_request_bytes: limits.maxRequestBytes }), + ...(backend.multipart === undefined + ? {} + : { multipart_part_bytes: limits.multipartPartBytes }), + }, }); } if (operation === "list") { @@ -145,6 +171,7 @@ async function handleRequest( rejectUnexpectedQuery(url); const path = objectPath!; + enforceRequestLimit(input.request.headers, limits.maxRequestBytes); switch (operation) { case "raw": return raw(input.request, backend, path); @@ -156,6 +183,14 @@ async function handleRequest( return receivePack(input.request, backend, path); case "receive-bundle": return receiveBundle(input.request, backend, path); + case "multipart-start": + return startMultipartUpload(input.request, backend, path, limits.multipartPartBytes); + case "multipart-part": + return uploadMultipartPart(input.request, backend, path, limits.multipartPartBytes); + case "multipart-complete": + return completeMultipartUpload(input.request, backend, path); + case "multipart-abort": + return abortMultipartUpload(input.request, backend, path); case "cas": return compareAndSwap(input.request, backend, path); case "cad": @@ -222,6 +257,16 @@ function validateMethodAndAction( case "receive-bundle": requireMethod(method, "POST"); return "write"; + case "multipart-start": + case "multipart-complete": + requireMethod(method, "POST"); + return "write"; + case "multipart-part": + requireMethod(method, "PUT"); + return "write"; + case "multipart-abort": + requireMethod(method, "DELETE"); + return "write"; case "cas": case "cad": requireMethod(method, "POST"); @@ -229,6 +274,93 @@ function validateMethodAndAction( } } +async function startMultipartUpload( + request: Request, + backend: GraftRepositoryBackend, + path: string, + partBytes: number, +): Promise { + requireImmutablePath(path); + const multipart = requireMultipartBackend(backend); + requireEmptyBody(request); + const totalBytes = parsePositiveIntegerHeader(request.headers, MULTIPART_HEADER_OBJECT_BYTES); + const parts = Math.ceil(totalBytes / partBytes); + if (parts > MAX_MULTIPART_PARTS) { + throw new GraftProtocolError( + 413, + "multipart_object_too_large", + `Multipart object requires more than ${MAX_MULTIPART_PARTS} parts`, + ); + } + const upload = await multipart.start(path, totalBytes, partBytes); + if (upload === null) { + throw new GraftProtocolError(412, "precondition_failed", "Object already exists"); + } + validateMultipartUpload(upload, totalBytes, partBytes); + return jsonResponse({ + upload_id: upload.uploadId, + total_bytes: upload.totalBytes, + part_bytes: upload.partBytes, + uploaded_parts: upload.uploadedParts.map((part) => ({ + part_number: part.partNumber, + bytes: part.bytes, + })), + }); +} + +async function uploadMultipartPart( + request: Request, + backend: GraftRepositoryBackend, + path: string, + partBytes: number, +): Promise { + requireImmutablePath(path); + const multipart = requireMultipartBackend(backend); + const uploadId = parseUploadId(request.headers); + const partNumber = parsePositiveIntegerHeader(request.headers, MULTIPART_HEADER_PART_NUMBER); + if (partNumber > MAX_MULTIPART_PARTS) { + throw new GraftProtocolError(400, "invalid_multipart_part", "Multipart part number is too large"); + } + const contentLength = parseContentLengthHeader(request.headers); + if (contentLength < 1 || contentLength > partBytes) { + throw new GraftProtocolError( + 400, + "invalid_multipart_part", + "Multipart part Content-Length is outside the advertised part size", + ); + } + if (request.body === null) { + throw new GraftProtocolError(400, "invalid_multipart_part", "Multipart part body is missing"); + } + await multipart.uploadPart(path, uploadId, partNumber, request.body, contentLength); + return emptyResponse(); +} + +async function completeMultipartUpload( + request: Request, + backend: GraftRepositoryBackend, + path: string, +): Promise { + requireImmutablePath(path); + requireEmptyBody(request); + const created = await requireMultipartBackend(backend).complete(path, parseUploadId(request.headers)); + if (!created) { + throw new GraftProtocolError(412, "precondition_failed", "Object already exists"); + } + return emptyResponse(); +} + +async function abortMultipartUpload( + request: Request, + backend: GraftRepositoryBackend, + path: string, +): Promise { + requireImmutablePath(path); + requireEmptyBody(request); + await requireMultipartBackend(backend).abort(path, parseUploadId(request.headers)); + return emptyResponse(); +} + async function raw( request: Request, backend: GraftRepositoryBackend, @@ -919,6 +1051,159 @@ async function listObjects(backend: GraftRepositoryBackend, url: URL): Promise["limits"], +): NormalizedRemoteLimits { + const maxRequestBytes = limits?.maxRequestBytes; + const multipartPartBytes = limits?.multipartPartBytes ?? DEFAULT_MULTIPART_PART_BYTES; + if ( + maxRequestBytes !== undefined && + (!Number.isSafeInteger(maxRequestBytes) || maxRequestBytes < 1) + ) { + throw new TypeError("maxRequestBytes must be a positive safe integer"); + } + if (!Number.isSafeInteger(multipartPartBytes) || multipartPartBytes < 1) { + throw new TypeError("multipartPartBytes must be a positive safe integer"); + } + if (maxRequestBytes !== undefined && multipartPartBytes > maxRequestBytes) { + throw new TypeError("multipartPartBytes cannot exceed maxRequestBytes"); + } + return { + ...(maxRequestBytes === undefined ? {} : { maxRequestBytes }), + multipartPartBytes, + }; +} + +function enforceRequestLimit(headers: Headers, maxRequestBytes: number | undefined): void { + if (maxRequestBytes === undefined) return; + const value = headers.get("content-length"); + if (value === null) return; + if (!/^(?:0|[1-9]\d*)$/.test(value) || !Number.isSafeInteger(Number(value))) { + throw new GraftProtocolError( + 400, + "invalid_content_length", + "Content-Length must be a non-negative safe integer", + ); + } + if (Number(value) > maxRequestBytes) { + throw new GraftProtocolError( + 413, + "request_too_large", + "Request exceeds the remote service request limit", + ); + } +} + +function requireMultipartBackend( + backend: GraftRepositoryBackend, +): NonNullable { + if (backend.multipart === undefined) { + throw new GraftProtocolError( + 404, + "operation_not_found", + "Multipart object upload is not supported", + ); + } + return backend.multipart; +} + +function requireImmutablePath(path: string): void { + if (!isImmutablePath(path)) { + throw new GraftProtocolError( + 400, + "invalid_immutable_path", + "Multipart upload is only defined for immutable objects", + ); + } +} + +function requireEmptyBody(request: Request): void { + const value = request.headers.get("content-length"); + if (value !== null && value !== "0") { + throw new GraftProtocolError(400, "unexpected_body", "Request body must be empty"); + } +} + +function parsePositiveIntegerHeader(headers: Headers, name: string): number { + const value = headers.get(name); + if (value === null || !/^[1-9]\d*$/.test(value)) { + throw new GraftProtocolError(400, "invalid_multipart_upload", `${name} must be positive`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new GraftProtocolError(413, "multipart_object_too_large", `${name} is too large`); + } + return parsed; +} + +function parseContentLengthHeader(headers: Headers): number { + const value = headers.get("content-length"); + if (value === null || !/^(?:0|[1-9]\d*)$/.test(value)) { + throw new GraftProtocolError( + 400, + "invalid_multipart_part", + "Multipart part requires Content-Length", + ); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new GraftProtocolError(413, "multipart_part_too_large", "Multipart part is too large"); + } + return parsed; +} + +function parseUploadId(headers: Headers): string { + const value = headers.get(MULTIPART_HEADER_UPLOAD_ID); + if ( + value === null || + value.length < 1 || + value.length > 1_024 || + /[\u0000-\u001f\u007f]/.test(value) + ) { + throw new GraftProtocolError( + 400, + "invalid_multipart_upload", + `${MULTIPART_HEADER_UPLOAD_ID} is invalid`, + ); + } + return value; +} + +function validateMultipartUpload( + upload: NonNullable< + Awaited["start"]>> + >, + totalBytes: number, + partBytes: number, +): void { + if ( + upload.uploadId.length < 1 || + upload.uploadId.length > 1_024 || + /[\u0000-\u001f\u007f]/.test(upload.uploadId) || + upload.totalBytes !== totalBytes || + upload.partBytes !== partBytes + ) { + throw backendContractError("Multipart backend returned an invalid upload session"); + } + const partCount = Math.ceil(totalBytes / partBytes); + let previous = 0; + for (const part of upload.uploadedParts) { + if ( + !Number.isSafeInteger(part.partNumber) || + part.partNumber <= previous || + part.partNumber > partCount + ) { + throw backendContractError("Multipart backend returned invalid uploaded parts"); + } + const expectedBytes = + part.partNumber === partCount ? totalBytes - partBytes * (partCount - 1) : partBytes; + if (part.bytes !== expectedBytes) { + throw backendContractError("Multipart backend returned an invalid uploaded part size"); + } + previous = part.partNumber; + } +} + function objectHeaders(metadata: GraftObjectMetadata): Headers { const headers = new Headers({ "Accept-Ranges": "bytes", diff --git a/packages/graft-remote/src/index.ts b/packages/graft-remote/src/index.ts index f4f74889..97aee018 100644 --- a/packages/graft-remote/src/index.ts +++ b/packages/graft-remote/src/index.ts @@ -5,6 +5,9 @@ export { MAX_LIST_LIMIT, MAX_METADATA_BYTES, MAX_UPLOAD_BUNDLE_OBJECTS, + MULTIPART_HEADER_OBJECT_BYTES, + MULTIPART_HEADER_PART_NUMBER, + MULTIPART_HEADER_UPLOAD_ID, PROTOCOL_HEADER, PROTOCOL_VERSION, RECEIVE_PACK_HEADER_INDEX_BYTES, @@ -24,6 +27,9 @@ export type { GraftHandlerRequest, GraftListQuery, GraftListResult, + GraftMultipartBackend, + GraftMultipartPart, + GraftMultipartUpload, GraftObject, GraftObjectBody, GraftObjectMetadata, diff --git a/packages/graft-remote/src/protocol.ts b/packages/graft-remote/src/protocol.ts index 0cb1136a..81a682af 100644 --- a/packages/graft-remote/src/protocol.ts +++ b/packages/graft-remote/src/protocol.ts @@ -24,6 +24,9 @@ export const RECEIVE_PACK_HEADER_PACK_BYTES = "x-graft-pack-bytes"; export const RECEIVE_PACK_HEADER_INDEX_BYTES = "x-graft-index-bytes"; export const RECEIVE_PACK_HEADER_REPLACEMENT_HEX = "x-graft-ref-replacement-hex"; export const RECEIVE_BUNDLE_HEADER_MANIFEST_BYTES = "x-graft-bundle-manifest-bytes"; +export const MULTIPART_HEADER_OBJECT_BYTES = "x-graft-object-bytes"; +export const MULTIPART_HEADER_UPLOAD_ID = "x-graft-upload-id"; +export const MULTIPART_HEADER_PART_NUMBER = "x-graft-part-number"; export const MAX_RECEIVE_BUNDLE_OBJECTS = 256; export const MAX_UPLOAD_BUNDLE_OBJECTS = 65_536; const REPOSITORY_SEGMENT = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,62}[A-Za-z0-9])?$/; diff --git a/packages/graft-remote/src/types.ts b/packages/graft-remote/src/types.ts index 1a65802a..27ecdcb3 100644 --- a/packages/graft-remote/src/types.ts +++ b/packages/graft-remote/src/types.ts @@ -40,6 +40,35 @@ export interface GraftListResult { hasMore: boolean; } +export interface GraftMultipartPart { + partNumber: number; + bytes: number; +} + +export interface GraftMultipartUpload { + uploadId: string; + totalBytes: number; + partBytes: number; + uploadedParts: GraftMultipartPart[]; +} + +export interface GraftMultipartBackend { + start( + path: string, + totalBytes: number, + partBytes: number, + ): MaybePromise; + uploadPart( + path: string, + uploadId: string, + partNumber: number, + value: ReadableStream, + contentLength: number, + ): MaybePromise; + complete(path: string, uploadId: string): MaybePromise; + abort(path: string, uploadId: string): MaybePromise; +} + export interface GraftRepositoryBackend { head(path: string): MaybePromise; get(path: string, range?: GraftByteRange): MaybePromise; @@ -61,6 +90,7 @@ export interface GraftRepositoryBackend { expected: Uint8Array | undefined, ): MaybePromise; list(query: GraftListQuery): MaybePromise; + multipart?: GraftMultipartBackend; } export type GraftRemoteAction = "discover" | "read" | "write"; @@ -72,6 +102,10 @@ export type GraftRemoteOperation = | "upload-bundle" | "receive-pack" | "receive-bundle" + | "multipart-start" + | "multipart-part" + | "multipart-complete" + | "multipart-abort" | "cas" | "cad" | "list"; @@ -110,6 +144,10 @@ export interface GraftRemoteOptions, ): MaybePromise; onError?(error: unknown, request: GraftHandlerRequest): MaybePromise; + limits?: { + maxRequestBytes?: number; + multipartPartBytes?: number; + }; } export type GraftRemoteHandler = ( diff --git a/packages/graft-remote/test/remote.test.ts b/packages/graft-remote/test/remote.test.ts index 12e8d37d..ba5bf880 100644 --- a/packages/graft-remote/test/remote.test.ts +++ b/packages/graft-remote/test/remote.test.ts @@ -7,6 +7,7 @@ import { createGraftRemoteHandler, type GraftByteRange, type GraftListQuery, + type GraftMultipartBackend, type GraftObject, type GraftObjectMetadata, type GraftRepositoryBackend, @@ -17,6 +18,66 @@ const ORIGIN = "https://remote.example"; class MemoryRepository implements GraftRepositoryBackend { readonly objects = new Map>(); + readonly uploads = new Map< + string, + { + uploadId: string; + totalBytes: number; + partBytes: number; + parts: Map>; + } + >(); + readonly multipart: GraftMultipartBackend = { + start: (path, totalBytes, partBytes) => { + if (this.objects.has(path)) return null; + const existing = this.uploads.get(path); + if (existing !== undefined) { + return { + uploadId: existing.uploadId, + totalBytes: existing.totalBytes, + partBytes: existing.partBytes, + uploadedParts: [...existing.parts] + .sort(([left], [right]) => left - right) + .map(([partNumber, bytes]) => ({ partNumber, bytes: bytes.byteLength })), + }; + } + const upload = { + uploadId: crypto.randomUUID(), + totalBytes, + partBytes, + parts: new Map>(), + }; + this.uploads.set(path, upload); + return { uploadId: upload.uploadId, totalBytes, partBytes, uploadedParts: [] }; + }, + uploadPart: async (path, uploadId, partNumber, value) => { + const upload = this.uploads.get(path); + if (upload === undefined || upload.uploadId !== uploadId) { + throw new RangeError("unknown multipart upload"); + } + upload.parts.set(partNumber, await bodyBytes(value)); + }, + complete: (path, uploadId) => { + if (this.objects.has(path)) return false; + const upload = this.uploads.get(path); + if (upload === undefined || upload.uploadId !== uploadId) { + throw new RangeError("unknown multipart upload"); + } + const value = new Uint8Array(upload.totalBytes); + let offset = 0; + for (const [, part] of [...upload.parts].sort(([left], [right]) => left - right)) { + value.set(part, offset); + offset += part.byteLength; + } + if (offset !== value.byteLength) throw new RangeError("incomplete multipart upload"); + this.objects.set(path, value); + this.uploads.delete(path); + return true; + }, + abort: (path, uploadId) => { + if (this.uploads.get(path)?.uploadId === uploadId) this.uploads.delete(path); + }, + }; head(path: string): GraftObjectMetadata | null { const value = this.objects.get(path); @@ -90,7 +151,7 @@ class MemoryRepository implements GraftRepositoryBackend { } } -function createTestApp() { +function createTestApp(limits?: { maxRequestBytes?: number; multipartPartBytes?: number }) { const repositories = new Map(); return createGraftRemoteHandler({ authenticate({ request }) { @@ -108,6 +169,7 @@ function createTestApp() { } return backend; }, + limits, }); } @@ -154,6 +216,86 @@ describe("createGraftRemoteHandler", () => { }); }); + it("resumes multipart immutable uploads and advertises request limits", async () => { + const app = createTestApp({ maxRequestBytes: 8, multipartPartBytes: 5 }); + const descriptor = await remoteFetch(app, "/acme/archive"); + expect(await descriptor.json()).toMatchObject({ + capabilities: expect.arrayContaining(["multipart-object"]), + limits: { max_request_bytes: 8, multipart_part_bytes: 5 }, + }); + + const start = await remoteFetch(app, "/acme/archive/multipart-start/segments/large", { + method: "POST", + headers: { "x-graft-object-bytes": "11", "Content-Length": "0" }, + }); + expect(start.status).toBe(200); + const session = (await start.json()) as { upload_id: string }; + + expect( + ( + await remoteFetch(app, "/acme/archive/multipart-part/segments/large", { + method: "PUT", + headers: { + "x-graft-upload-id": session.upload_id, + "x-graft-part-number": "1", + "Content-Length": "5", + }, + body: "abcde", + }) + ).status, + ).toBe(204); + + const resumed = await remoteFetch(app, "/acme/archive/multipart-start/segments/large", { + method: "POST", + headers: { "x-graft-object-bytes": "11", "Content-Length": "0" }, + }); + expect(await resumed.json()).toMatchObject({ + upload_id: session.upload_id, + uploaded_parts: [{ part_number: 1, bytes: 5 }], + }); + + for (const [partNumber, body] of [ + [2, "fghij"], + [3, "k"], + ] as const) { + const part = await remoteFetch(app, "/acme/archive/multipart-part/segments/large", { + method: "PUT", + headers: { + "x-graft-upload-id": session.upload_id, + "x-graft-part-number": partNumber.toString(), + "Content-Length": body.length.toString(), + }, + body, + }); + expect(part.status).toBe(204); + } + + const complete = await remoteFetch(app, "/acme/archive/multipart-complete/segments/large", { + method: "POST", + headers: { "x-graft-upload-id": session.upload_id, "Content-Length": "0" }, + }); + expect(complete.status).toBe(204); + expect( + await (await remoteFetch(app, "/acme/archive/raw/segments/large")).text(), + ).toBe("abcdefghijk"); + expect( + ( + await remoteFetch(app, "/acme/archive/multipart-start/segments/large", { + method: "POST", + headers: { "x-graft-object-bytes": "11", "Content-Length": "0" }, + }) + ).status, + ).toBe(412); + + const tooLarge = await remoteFetch(app, "/acme/archive/raw-if-not-exists/segments/direct", { + method: "PUT", + headers: { "Content-Length": "9" }, + body: "123456789", + }); + expect(tooLarge.status).toBe(413); + expect(await tooLarge.json()).toMatchObject({ title: "request_too_large" }); + }); + it("separates authentication, authorization, repository mapping, and storage", async () => { const backend = new MemoryRepository(); const authorized: Array<{ diff --git a/services/graft-remote-cloudflare/test/worker.test.ts b/services/graft-remote-cloudflare/test/worker.test.ts index b25c8ee4..42a73459 100644 --- a/services/graft-remote-cloudflare/test/worker.test.ts +++ b/services/graft-remote-cloudflare/test/worker.test.ts @@ -114,6 +114,54 @@ describe("graft remote protocol", () => { expect(head.headers.get("content-length")).toBe("6"); }); + it("resumes R2 multipart uploads without changing the immutable object path", async () => { + const totalBytes = 6 * 1024 * 1024; + const start = await remoteFetch("/multipart/repo/multipart-start/segments/large", { + method: "POST", + headers: { + "content-length": "0", + "x-graft-object-bytes": totalBytes.toString(), + }, + }); + expect(start.status).toBe(200); + const session = (await start.json()) as { upload_id: string }; + const part = await remoteFetch("/multipart/repo/multipart-part/segments/large", { + method: "PUT", + headers: { + "content-length": totalBytes.toString(), + "x-graft-upload-id": session.upload_id, + "x-graft-part-number": "1", + }, + body: new Uint8Array(totalBytes).fill(7), + }); + expect(part.status).toBe(204); + + const resumed = await remoteFetch("/multipart/repo/multipart-start/segments/large", { + method: "POST", + headers: { + "content-length": "0", + "x-graft-object-bytes": totalBytes.toString(), + }, + }); + expect(await resumed.json()).toMatchObject({ + upload_id: session.upload_id, + uploaded_parts: [{ part_number: 1, bytes: totalBytes }], + }); + const completed = await remoteFetch( + "/multipart/repo/multipart-complete/segments/large", + { + method: "POST", + headers: { + "content-length": "0", + "x-graft-upload-id": session.upload_id, + }, + }, + ); + expect(completed.status).toBe(204); + const head = await remoteFetch("/multipart/repo/raw/segments/large", { method: "HEAD" }); + expect(head.headers.get("content-length")).toBe(totalBytes.toString()); + }); + it("streams an upload bundle across transactional storage and R2", async () => { await remoteFetch("/upload/repo/raw/refs/heads/main", { method: "PUT", From 76a0219bf1c24f329f360d2a407cc453d2c73fec Mon Sep 17 00:00:00 2001 From: Mayne Date: Sat, 1 Aug 2026 18:32:54 +0800 Subject: [PATCH 2/4] perf(sqlite): accelerate sparse large-table diffs Use stable APFS worktree snapshots and page ownership to limit summary work to changed tables. Decode only changed WITHOUT ROWID B-tree pages for sparse edits and keep checkpoint fallbacks bounded when the live worktree advances after staging. --- crates/graft-sdk/src/lib.rs | 145 ++++++ crates/graft-sqlite/src/pragma/repo_diff.rs | 87 +++- crates/graft-sqlite/src/pragma/repo_output.rs | 27 +- .../src/pragma/sqlite_worktree.rs | 220 +++++++++ crates/graft-sqlite/src/row_level_diff.rs | 439 +++++++++++++++--- crates/graft-sqlite/src/sqlite_parse.rs | 116 +++++ 6 files changed, 954 insertions(+), 80 deletions(-) diff --git a/crates/graft-sdk/src/lib.rs b/crates/graft-sdk/src/lib.rs index 615820ab..9975f29c 100644 --- a/crates/graft-sdk/src/lib.rs +++ b/crates/graft-sdk/src/lib.rs @@ -3480,6 +3480,151 @@ mod tests { assert_eq!(error.code(), SdkErrorCode::Cancelled); } + #[test] + fn worktree_sqlite_diff_skips_large_tables_after_nullable_column_append() { + let directory = tempfile::tempdir().unwrap(); + let database_path = directory.path().join("space.eidos"); + let mut database = rusqlite::Connection::open(&database_path).unwrap(); + database + .execute_batch( + "CREATE TABLE archive ( + id TEXT PRIMARY KEY, + value TEXT NOT NULL + ) STRICT, WITHOUT ROWID; + CREATE TABLE notes ( + id TEXT PRIMARY KEY, + value TEXT NOT NULL + ) STRICT, WITHOUT ROWID;", + ) + .unwrap(); + let transaction = database.transaction().unwrap(); + for index in 0..20_000 { + transaction + .execute( + "INSERT INTO archive (id, value) VALUES (?1, ?2)", + [format!("archive-{index:05}"), format!("value-{index}")], + ) + .unwrap(); + } + transaction.commit().unwrap(); + database + .execute( + "INSERT INTO notes (id, value) VALUES ('note-0', 'baseline')", + [], + ) + .unwrap(); + drop(database); + + let session = RepositorySession::new(directory.path()); + session.open().unwrap(); + session.init().unwrap(); + session.add_all().unwrap(); + session.commit("large baseline").unwrap(); + + let database = rusqlite::Connection::open(&database_path).unwrap(); + database + .execute_batch( + "ALTER TABLE notes ADD COLUMN detail TEXT; + INSERT INTO notes (id, value) VALUES ('note-1', 'one'); + INSERT INTO notes (id, value, detail) VALUES ('note-2', 'two', 'new field');", + ) + .unwrap(); + drop(database); + + let options = |response| SqliteDiffPathsOptions { + paths: vec![PathBuf::from("space.eidos")], + root: None, + from: None, + to: None, + response, + limit: 1, + after: None, + }; + let summary = session + .diff_sqlite_paths(&options(SqliteDiffResponse::Summary)) + .unwrap(); + let file = &summary.paths[0].diff["files"][0]; + assert_eq!(file["summaries"].as_array().unwrap().len(), 1); + assert_eq!(file["summaries"][0]["name"], "notes"); + assert_eq!(file["summaries"][0]["inserts"], 2); + assert_eq!(summary.telemetry.tables_scanned, 1); + assert_eq!(summary.telemetry.response_scope, "streaming_primary_key"); + + let rows = session + .diff_sqlite_paths(&options(SqliteDiffResponse::Rows { + table: "notes".to_string(), + limit: 100, + after: None, + })) + .unwrap(); + assert_eq!(rows.telemetry.tables_scanned, 1); + assert_eq!(rows.telemetry.rows_returned, 2); + assert_eq!(rows.telemetry.response_scope, "streaming_primary_key"); + assert_eq!( + rows.paths[0].diff["files"][0]["tables"][0]["changes"][1]["values"][2], + "new field" + ); + + session + .stage_paths(&StagePathsOptions { + paths: vec![PathBuf::from("space.eidos")], + expected_head: None, + force: false, + }) + .unwrap(); + session.commit("small table change").unwrap(); + let history = session.history_summaries(1, None).unwrap(); + assert_eq!(history.commits[0].changed_tables, 1); + assert_eq!(history.commits[0].tables[0].name, "notes"); + assert_eq!(history.commits[0].tables[0].inserts, 2); + + let database = rusqlite::Connection::open(&database_path).unwrap(); + database + .execute_batch( + "DELETE FROM archive WHERE id = 'archive-00001'; + UPDATE archive SET value = 'changed' WHERE id = 'archive-10000'; + INSERT INTO archive (id, value) VALUES ('archive-99999', 'new');", + ) + .unwrap(); + drop(database); + + let summary = session + .diff_sqlite_paths(&options(SqliteDiffResponse::Summary)) + .unwrap(); + let file = &summary.paths[0].diff["files"][0]; + assert_eq!(file["summaries"].as_array().unwrap().len(), 1); + assert_eq!(file["summaries"][0]["name"], "archive"); + assert_eq!(file["summaries"][0]["inserts"], 1); + assert_eq!(file["summaries"][0]["deletes"], 1); + assert_eq!(file["summaries"][0]["updates"], 1); + assert!(summary.telemetry.rows_scanned < 2_000); + + let rows = session + .diff_sqlite_paths(&options(SqliteDiffResponse::Rows { + table: "archive".to_string(), + limit: 100, + after: None, + })) + .unwrap(); + assert_eq!(rows.telemetry.rows_returned, 3); + assert!(rows.telemetry.rows_scanned < 2_000); + + session + .stage_paths(&StagePathsOptions { + paths: vec![PathBuf::from("space.eidos")], + expected_head: None, + force: false, + }) + .unwrap(); + session.commit("large table sparse change").unwrap(); + let history = session.history_summaries(1, None).unwrap(); + assert_eq!(history.commits[0].changed_tables, 1); + assert_eq!(history.commits[0].tables[0].name, "archive"); + assert_eq!(history.commits[0].tables[0].inserts, 1); + assert_eq!(history.commits[0].tables[0].deletes, 1); + assert_eq!(history.commits[0].tables[0].updates, 1); + } + #[test] fn concurrent_path_type_churn_never_poison_session() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/graft-sqlite/src/pragma/repo_diff.rs b/crates/graft-sqlite/src/pragma/repo_diff.rs index 71947c93..f5957575 100644 --- a/crates/graft-sqlite/src/pragma/repo_diff.rs +++ b/crates/graft-sqlite/src/pragma/repo_diff.rs @@ -580,7 +580,7 @@ pub(super) fn staged_commit_table_summary( let diff = repo.diff_staged(None)?; let mut by_name = BTreeMap::::new(); for file in &diff.files { - let summaries = repo_file_table_summary(runtime, file)?; + let summaries = repo_file_table_summary(runtime, repo, file)?; for summary in summaries { merge_table_summary(&mut by_name, summary); } @@ -590,6 +590,7 @@ pub(super) fn staged_commit_table_summary( pub(super) fn repo_file_table_summary( runtime: &Runtime, + repo: &Repository, file: &graft::repo::RepoFileDiff, ) -> Result, ErrCtx> { match (&file.from, &file.to) { @@ -610,18 +611,33 @@ pub(super) fn repo_file_table_summary( SnapshotSummaryMode::Deleted, ); } - let diff = crate::row_level_diff::row_level_diff_snapshots( - runtime, - &from_snapshot, - &to_snapshot, + if let Some(summaries) = + staged_worktree_table_summary(runtime, repo, file, &from_snapshot, &to_snapshot)? + { + return Ok(summaries); + } + let from_reader = runtime.snapshot_reader(from_snapshot.clone()); + let to_reader = runtime.snapshot_reader(to_snapshot.clone()); + let from_lsn = from_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let to_lsn = to_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let diff = crate::row_level_diff::bounded_row_level_diff_readers( + &from_reader, + &to_reader, + from_lsn, + to_lsn, + &crate::row_level_diff::BoundedRowDiffMode::Summary, ) - .map_err(|e| ErrCtx::PragmaErr(format!("Diff error: {e:?}").into()))?; + .map_err(|error| ErrCtx::PragmaErr(format!("Diff error: {error:?}").into()))?; Ok(diff - .table_changes - .iter() - .filter_map(|table| { - let (inserts, deletes, updates) = count_changes_json(&table.changes); - table_summary(table.table_name.clone(), inserts, deletes, updates) + .summaries + .into_iter() + .filter_map(|summary| { + table_summary( + summary.table_name, + summary.inserts, + summary.deletes, + summary.updates, + ) }) .collect()) } @@ -639,6 +655,55 @@ pub(super) fn repo_file_table_summary( } } +fn staged_worktree_table_summary( + runtime: &Runtime, + repo: &Repository, + file: &graft::repo::RepoFileDiff, + from_snapshot: &graft::snapshot::Snapshot, + to_snapshot: &graft::snapshot::Snapshot, +) -> Result>, ErrCtx> { + let physical_path = repo.worktree().join(&file.path); + let metadata = match std::fs::symlink_metadata(&physical_path) { + Ok(metadata) if metadata.file_type().is_file() => metadata, + Ok(_) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if metadata.len() < 100 || !is_sqlite_database_path(&physical_path)? { + return Ok(None); + } + + let physical = PhysicalSqliteReader::open(&physical_path)?; + let from_reader = runtime.snapshot_reader(from_snapshot.clone()); + let to_reader = runtime.snapshot_reader(to_snapshot.clone()); + let Some(tables) = physical.staged_table_candidates(&to_reader, &from_reader)? else { + return Ok(None); + }; + let from_lsn = from_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let to_lsn = to_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let diff = crate::row_level_diff::bounded_row_level_diff_readers_for_summary_tables( + &from_reader, + &to_reader, + from_lsn, + to_lsn, + &tables, + ) + .map_err(|error| ErrCtx::PragmaErr(format!("Diff error: {error:?}").into()))?; + Ok(Some( + diff.summaries + .into_iter() + .filter_map(|summary| { + table_summary( + summary.table_name, + summary.inserts, + summary.deletes, + summary.updates, + ) + }) + .collect(), + )) +} + #[derive(Debug, Clone, Copy)] pub(super) enum SnapshotSummaryMode { Inserted, diff --git a/crates/graft-sqlite/src/pragma/repo_output.rs b/crates/graft-sqlite/src/pragma/repo_output.rs index cf8a86d4..14f5ef53 100644 --- a/crates/graft-sqlite/src/pragma/repo_output.rs +++ b/crates/graft-sqlite/src/pragma/repo_output.rs @@ -290,13 +290,26 @@ pub(super) fn repo_file_bounded_row_diff( let snapshot = from.snapshot.to_snapshot(); let from_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); let from_reader = runtime.snapshot_reader(snapshot); - crate::row_level_diff::bounded_row_level_diff_readers( - &from_reader, - &physical, - from_lsn, - from_lsn.saturating_next(), - mode, - ) + let summary_tables = matches!(mode, crate::row_level_diff::BoundedRowDiffMode::Summary) + .then(|| physical.changed_table_candidates(&from_reader)) + .transpose()?; + if let Some(Some(tables)) = summary_tables { + crate::row_level_diff::bounded_row_level_diff_readers_for_summary_tables( + &from_reader, + &physical, + from_lsn, + from_lsn.saturating_next(), + &tables, + ) + } else { + crate::row_level_diff::bounded_row_level_diff_readers( + &from_reader, + &physical, + from_lsn, + from_lsn.saturating_next(), + mode, + ) + } } else { let empty = empty_sqlite_reader()?; crate::row_level_diff::bounded_row_level_diff_readers( diff --git a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs index c65af4d6..1847c353 100644 --- a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs +++ b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs @@ -1,6 +1,11 @@ use graft::volume_writer::VolumeWriter; use rusqlite::{Connection, ErrorCode, OpenFlags, backup::Backup}; use std::time::{Duration, Instant}; +#[cfg(target_os = "macos")] +use std::{ + ffi::{CString, c_char, c_int}, + os::unix::ffi::OsStrExt, +}; use tempfile::TempDir; use super::*; @@ -12,6 +17,7 @@ use super::*; pub(super) struct PhysicalSqliteReader { input: Mutex, path: PathBuf, + snapshot_path: PathBuf, snapshot: graft::snapshot::Snapshot, _snapshot_dir: Option, } @@ -71,6 +77,7 @@ impl PhysicalSqliteReader { Ok(Self { input: Mutex::new(input), path: path.to_path_buf(), + snapshot_path: snapshot_path.to_path_buf(), snapshot, _snapshot_dir: snapshot_dir, }) @@ -80,6 +87,122 @@ impl PhysicalSqliteReader { RepoWorktreeFileState { page_count: self.page_count() } } + /// Finds tables that own pages changed from `expected` to this stable worktree snapshot. + /// + /// `dbstat` gives us `SQLite`'s page ownership without decoding every row. Returning `None` + /// deliberately falls back to the full logical scan whenever a changed page cannot be mapped, + /// preserving exact diff semantics for unusual freelist or extension layouts. + pub(super) fn changed_table_candidates( + &self, + expected: &dyn VolumeRead, + ) -> Result>, ErrCtx> { + let max_page_count = self + .page_count() + .to_u32() + .max(expected.page_count().to_u32()); + let mut changed_pages = BTreeSet::new(); + for page_number in 1..=max_page_count { + if page_number.is_multiple_of(1_024) { + graft::repo::cancellation_checkpoint()?; + } + let pageidx = PageIdx::try_from(page_number).map_err(|error| { + ErrCtx::PragmaErr( + format!("invalid SQLite page index {page_number}: {error}").into(), + ) + })?; + if self.read_page(pageidx)? != expected.read_page(pageidx)? { + changed_pages.insert(page_number); + } + } + self.table_candidates_for_changed_pages(&changed_pages) + } + + /// Reuses the physical worktree as a fast staged-snapshot reader only when every byte still + /// matches the staged state. While validating that invariant, collect pages changed from the + /// previous commit so checkpoint summaries can avoid scanning unrelated large tables. + pub(super) fn staged_table_candidates( + &self, + staged: &dyn VolumeRead, + previous: &dyn VolumeRead, + ) -> Result>, ErrCtx> { + if self.page_count() != staged.page_count() { + return Ok(None); + } + + let max_page_count = self + .page_count() + .to_u32() + .max(previous.page_count().to_u32()); + let mut changed_pages = BTreeSet::new(); + for page_number in 1..=max_page_count { + if page_number.is_multiple_of(1_024) { + graft::repo::cancellation_checkpoint()?; + } + let pageidx = PageIdx::try_from(page_number).map_err(|error| { + ErrCtx::PragmaErr( + format!("invalid SQLite page index {page_number}: {error}").into(), + ) + })?; + let physical_page = self.read_page(pageidx)?; + if page_number <= self.page_count().to_u32() + && physical_page != staged.read_page(pageidx)? + { + return Ok(None); + } + if physical_page != previous.read_page(pageidx)? { + changed_pages.insert(page_number); + } + } + self.table_candidates_for_changed_pages(&changed_pages) + } + + fn table_candidates_for_changed_pages( + &self, + changed_pages: &BTreeSet, + ) -> Result>, ErrCtx> { + if changed_pages.is_empty() { + return Ok(Some(BTreeSet::new())); + } + + let connection = Connection::open_with_flags( + &self.snapshot_path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + let mut statement = match connection.prepare( + "SELECT d.pageno, COALESCE(m.tbl_name, d.name) \ + FROM dbstat AS d \ + LEFT JOIN sqlite_schema AS m ON m.name = d.name", + ) { + Ok(statement) => statement, + Err(_) => return Ok(None), + }; + let mut mapped_pages = BTreeSet::new(); + let mut tables = BTreeSet::new(); + let rows = statement.query_map([], |row| { + Ok((row.get::<_, u32>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + let (page_number, table_name) = row?; + if !changed_pages.contains(&page_number) { + continue; + } + mapped_pages.insert(page_number); + if !table_name.starts_with("sqlite_") { + tables.insert(table_name); + } + } + + // Page 1 always carries SQLite's change counter and schema cookie. Schema changes are + // compared separately, so it is the only unmapped page that is safe to ignore here. + if changed_pages + .iter() + .any(|page_number| *page_number != 1 && !mapped_pages.contains(page_number)) + { + return Ok(None); + } + Ok(Some(tables)) + } + pub(super) fn matches_state( &self, runtime: &Runtime, @@ -152,6 +275,11 @@ fn backup_sqlite_source(path: &Path, snapshot_path: &Path) -> Result<(), ErrCtx> const BACKUP_RETRY_DELAY: Duration = Duration::from_millis(10); const PAGES_PER_STEP: i32 = 256; + #[cfg(target_os = "macos")] + if clone_rollback_journal_snapshot(path, snapshot_path, BACKUP_TIMEOUT)? { + return Ok(()); + } + let source = Connection::open_with_flags( path, OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, @@ -199,6 +327,58 @@ fn backup_sqlite_source(path: &Path, snapshot_path: &Path) -> Result<(), ErrCtx> Ok(()) } +#[cfg(target_os = "macos")] +fn clone_rollback_journal_snapshot( + path: &Path, + snapshot_path: &Path, + timeout: Duration, +) -> Result { + let source = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + source.busy_timeout(timeout)?; + let journal_mode: String = source.pragma_query_value(None, "journal_mode", |row| row.get(0))?; + if !journal_mode.eq_ignore_ascii_case("delete") { + return Ok(false); + } + + // A read transaction holds SQLite's shared lock while APFS creates the clone. Writers may + // continue preparing a transaction, but cannot publish an in-place rollback-journal commit + // until the clone has captured one coherent database image. + source.execute_batch("BEGIN")?; + source.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; + let clone_result = clone_file(path, snapshot_path); + source.execute_batch("ROLLBACK")?; + match clone_result { + Ok(()) => Ok(true), + Err(_) => { + let _ = std::fs::remove_file(snapshot_path); + Ok(false) + } + } +} + +#[cfg(target_os = "macos")] +fn clone_file(source: &Path, destination: &Path) -> std::io::Result<()> { + unsafe extern "C" { + fn clonefile(source: *const c_char, destination: *const c_char, flags: c_int) -> c_int; + } + + let source = CString::new(source.as_os_str().as_bytes()) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + let destination = CString::new(destination.as_os_str().as_bytes()) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + // SAFETY: both pointers come from live NUL-terminated `CString`s and flags=0 is the documented + // clonefile mode. The destination does not exist inside our private temporary directory. + let result = unsafe { clonefile(source.as_ptr(), destination.as_ptr(), 0) }; + if result == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + impl VolumeRead for PhysicalSqliteReader { fn snapshot(&self) -> &graft::snapshot::Snapshot { &self.snapshot @@ -617,6 +797,46 @@ mod tests { assert!(latest.changed_pages < updated.snapshot.page_count.to_u32() as usize); } + #[test] + fn staged_candidates_require_an_exact_worktree_match() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let updated = import_physical_sqlite_file_state(&runtime, &path, Some(&initial)).unwrap(); + let previous_reader = runtime.snapshot_reader(initial.snapshot.to_snapshot()); + let staged_reader = runtime.snapshot_reader(updated.snapshot.to_snapshot()); + let physical = PhysicalSqliteReader::open(&path).unwrap(); + assert_eq!( + physical + .staged_table_candidates(&staged_reader, &previous_reader) + .unwrap(), + Some(BTreeSet::from(["records".to_string()])) + ); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + let changed_after_stage = PhysicalSqliteReader::open(&path).unwrap(); + assert_eq!( + changed_after_stage + .staged_table_candidates(&staged_reader, &previous_reader) + .unwrap(), + None + ); + } + #[test] fn wal_import_reads_committed_state_without_checkpointing_source() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/graft-sqlite/src/row_level_diff.rs b/crates/graft-sqlite/src/row_level_diff.rs index 2a2f37f9..3b29a4ba 100644 --- a/crates/graft-sqlite/src/row_level_diff.rs +++ b/crates/graft-sqlite/src/row_level_diff.rs @@ -778,7 +778,26 @@ pub fn bounded_row_level_diff_readers( to_lsn: LSN, mode: &BoundedRowDiffMode, ) -> Result { - bounded_row_diff_from_readers(from_reader, to_reader, from_lsn, to_lsn, mode) + bounded_row_diff_from_readers(from_reader, to_reader, from_lsn, to_lsn, mode, None) +} + +/// Computes a summary after a worktree page comparison has already narrowed the candidate tables. +/// Schema changes are still included even when their table no longer owns a page in the worktree. +pub fn bounded_row_level_diff_readers_for_summary_tables( + from_reader: &dyn VolumeRead, + to_reader: &dyn VolumeRead, + from_lsn: LSN, + to_lsn: LSN, + tables: &BTreeSet, +) -> Result { + bounded_row_diff_from_readers( + from_reader, + to_reader, + from_lsn, + to_lsn, + &BoundedRowDiffMode::Summary, + Some(tables), + ) } fn bounded_row_diff_from_readers( @@ -787,6 +806,7 @@ fn bounded_row_diff_from_readers( from_lsn: LSN, to_lsn: LSN, mode: &BoundedRowDiffMode, + summary_tables: Option<&BTreeSet>, ) -> Result { let native_page_size = PAGESIZE.as_u32(); let needs_materialized_schema = sqlite_page_size(from_reader)? != native_page_size @@ -831,6 +851,12 @@ fn bounded_row_diff_from_readers( for entry in from_master.iter().chain(&to_master) { if is_diffable_table(entry, &ignored_tables) && requested_table.is_none_or(|table| entry.name == table) + && summary_tables.is_none_or(|tables| { + tables.contains(&entry.name) + || schema_changes + .iter() + .any(|change| change.name == entry.name) + }) { table_names.insert(entry.name.clone()); } @@ -863,7 +889,7 @@ fn bounded_row_diff_from_readers( || to_entry.is_some_and(is_without_rowid_table); let direct_primary_key = !needs_materialized_schema && without_rowid - && compatible_without_rowid_layout(from_entry, to_entry).is_some(); + && compatible_without_rowid_layouts(from_entry, to_entry).is_some(); let needs_sqlite_rows = needs_materialized_schema || (without_rowid && !direct_primary_key); let page = if needs_sqlite_rows { used_materialized_compat = true; @@ -1118,20 +1144,28 @@ impl WithoutRowidLayout { }) } - fn decode(&self, physical: Record) -> Result<(RowIdentity, Record), graft::err::GraftErr> { - if physical.values.len() != self.physical_to_declared.len() { + fn decode( + &self, + physical: Record, + output_columns: &[String], + ) -> Result<(RowIdentity, Record), graft::err::GraftErr> { + // SQLite does not rewrite existing records after `ALTER TABLE ADD COLUMN`, so records in + // the newer B-tree may legitimately omit trailing nullable columns from the newer schema. + if physical.values.len() > self.physical_to_declared.len() { return Err(graft::err::LogicalErr::Other(format!( - "WITHOUT ROWID record contains {} values, expected {}", + "WITHOUT ROWID record contains {} values, expected at most {}", physical.values.len(), self.physical_to_declared.len() )) .into()); } let mut values = vec![Value::Null; self.columns.len()]; - for (physical_index, declared_index) in - self.physical_to_declared.iter().copied().enumerate() + for (physical_value, declared_index) in physical + .values + .into_iter() + .zip(self.physical_to_declared.iter().copied()) { - values[declared_index] = physical.values[physical_index].clone(); + values[declared_index] = physical_value; } let key = self .primary_key_columns @@ -1148,10 +1182,30 @@ impl WithoutRowidLayout { } }) .collect(); - Ok((RowIdentity::PrimaryKey(key), Record { values })) + let output_values = output_columns + .iter() + .map(|column| { + self.columns + .iter() + .position(|candidate| candidate == column) + .map_or(Value::Null, |index| values[index].clone()) + }) + .collect(); + Ok(( + RowIdentity::PrimaryKey(key), + Record { values: output_values }, + )) } } +#[derive(Debug, Clone)] +struct WithoutRowidPairLayout { + from: Option, + to: Option, + output_columns: Vec, + primary_key_columns: Vec, +} + fn supports_direct_primary_key_order(entry: &MasterEntry, primary_key_columns: &[String]) -> bool { let column_definitions = parse_create_table_column_definitions(&entry.sql); let definitions_are_supported = primary_key_columns.iter().all(|primary_key| { @@ -1194,10 +1248,10 @@ fn ordering_fragment_is_supported(fragment: &str) -> bool { }) } -fn compatible_without_rowid_layout( +fn compatible_without_rowid_layouts( from_entry: Option<&MasterEntry>, to_entry: Option<&MasterEntry>, -) -> Option { +) -> Option { let from = match from_entry { Some(entry) => Some(WithoutRowidLayout::from_entry(entry)?), None => None, @@ -1206,17 +1260,60 @@ fn compatible_without_rowid_layout( Some(entry) => Some(WithoutRowidLayout::from_entry(entry)?), None => None, }; - match (from, to) { - (Some(from), Some(to)) - if from.columns == to.columns - && from.primary_key_columns == to.primary_key_columns - && from.physical_to_declared == to.physical_to_declared => - { - Some(to) - } - (Some(layout), None) | (None, Some(layout)) => Some(layout), - _ => None, + let primary_key_columns = to + .as_ref() + .or(from.as_ref()) + .map(|layout| layout.primary_key_columns.clone())?; + if from.as_ref().zip(to.as_ref()).is_some_and(|(from, to)| { + from.primary_key_columns != to.primary_key_columns + || !supports_nullable_appended_transition(from_entry, to_entry, from, to) + }) { + return None; + } + let output_columns = to + .as_ref() + .or(from.as_ref()) + .map(|layout| layout.columns.clone())?; + Some(WithoutRowidPairLayout { + from, + to, + output_columns, + primary_key_columns, + }) +} + +fn supports_nullable_appended_transition( + from_entry: Option<&MasterEntry>, + to_entry: Option<&MasterEntry>, + from: &WithoutRowidLayout, + to: &WithoutRowidLayout, +) -> bool { + if from.columns == to.columns { + return true; + } + let (shorter, longer, longer_entry) = if from.columns.len() < to.columns.len() { + (from, to, to_entry) + } else { + (to, from, from_entry) + }; + if !longer.columns.starts_with(&shorter.columns) { + return false; } + let Some(longer_entry) = longer_entry else { + return false; + }; + let definitions = parse_create_table_column_definitions(&longer_entry.sql); + longer.columns[shorter.columns.len()..] + .iter() + .all(|column| { + definitions + .iter() + .find(|definition| definition.name == *column) + .is_some_and(|definition| { + let sql = definition.sql.to_ascii_uppercase(); + !sql.contains("NOT NULL") && !sql.contains("DEFAULT") + }) + }) } fn bounded_without_rowid_table( @@ -1226,7 +1323,7 @@ fn bounded_without_rowid_table( to_entry: Option<&MasterEntry>, mode: &BoundedRowDiffMode, ) -> Result { - let layout = compatible_without_rowid_layout(from_entry, to_entry).ok_or_else(|| { + let layouts = compatible_without_rowid_layouts(from_entry, to_entry).ok_or_else(|| { graft::err::LogicalErr::Other("unsupported WITHOUT ROWID primary-key layout".into()) })?; let from_count = from_entry @@ -1247,18 +1344,33 @@ fn bounded_without_rowid_table( )) })? .unwrap_or(0); - if matches!(mode, BoundedRowDiffMode::Summary) && (from_count == 0 || to_count == 0) { + if matches!(mode, BoundedRowDiffMode::Summary) + && (from_entry.is_none() || to_entry.is_none() || from_count == 0 || to_count == 0) + { return Ok(BoundedTablePage { inserts: to_count, deletes: from_count, updates: 0, rows_scanned: 0, - primary_key_columns: layout.primary_key_columns, + primary_key_columns: layouts.primary_key_columns, changes: Vec::new(), has_more: false, next_offset: None, }); } + if from_count > 0 + && to_count > 0 + && let (Some(from_entry), Some(to_entry)) = (from_entry, to_entry) + { + return bounded_without_rowid_changed_pages( + from_reader, + to_reader, + from_entry, + to_entry, + &layouts, + mode, + ); + } let mut from_stream = from_entry .map(|entry| TableScanner::new(from_reader)?.into_index_row_stream(entry.root_page)) .transpose() @@ -1275,14 +1387,19 @@ fn bounded_without_rowid_table( "Failed to stream target WITHOUT ROWID rows: {error}" )) })?; - let mut from_row = next_bounded_primary_key_row(&mut from_stream, &layout)?; - let mut to_row = next_bounded_primary_key_row(&mut to_stream, &layout)?; + let mut from_row = next_bounded_primary_key_row( + &mut from_stream, + layouts.from.as_ref(), + &layouts.output_columns, + )?; + let mut to_row = + next_bounded_primary_key_row(&mut to_stream, layouts.to.as_ref(), &layouts.output_columns)?; let mut page = BoundedTablePage { inserts: 0, deletes: 0, updates: 0, rows_scanned: 0, - primary_key_columns: layout.primary_key_columns.clone(), + primary_key_columns: layouts.primary_key_columns.clone(), changes: Vec::new(), has_more: false, next_offset: None, @@ -1301,8 +1418,16 @@ fn bounded_without_rowid_table( if from_identity == to_identity => { page.rows_scanned = page.rows_scanned.saturating_add(2); - from_row = next_bounded_primary_key_row(&mut from_stream, &layout)?; - to_row = next_bounded_primary_key_row(&mut to_stream, &layout)?; + from_row = next_bounded_primary_key_row( + &mut from_stream, + layouts.from.as_ref(), + &layouts.output_columns, + )?; + to_row = next_bounded_primary_key_row( + &mut to_stream, + layouts.to.as_ref(), + &layouts.output_columns, + )?; (old_row != new_row).then(|| match from_identity { RowIdentity::PrimaryKey(key) => { RowChange::PrimaryKeyUpdate { key, old_row, new_row } @@ -1316,7 +1441,11 @@ fn bounded_without_rowid_table( if from_identity < to_identity => { page.rows_scanned = page.rows_scanned.saturating_add(1); - from_row = next_bounded_primary_key_row(&mut from_stream, &layout)?; + from_row = next_bounded_primary_key_row( + &mut from_stream, + layouts.from.as_ref(), + &layouts.output_columns, + )?; to_row = Some((to_identity, new_row)); match from_identity { RowIdentity::PrimaryKey(key) => { @@ -1330,7 +1459,11 @@ fn bounded_without_rowid_table( (Some((from_identity, old_row)), Some((to_identity, new_row))) => { page.rows_scanned = page.rows_scanned.saturating_add(1); from_row = Some((from_identity, old_row)); - to_row = next_bounded_primary_key_row(&mut to_stream, &layout)?; + to_row = next_bounded_primary_key_row( + &mut to_stream, + layouts.to.as_ref(), + &layouts.output_columns, + )?; match to_identity { RowIdentity::PrimaryKey(key) => { Some(RowChange::PrimaryKeyInsert { key, row: new_row }) @@ -1342,7 +1475,11 @@ fn bounded_without_rowid_table( } (Some((identity, row)), None) => { page.rows_scanned = page.rows_scanned.saturating_add(1); - from_row = next_bounded_primary_key_row(&mut from_stream, &layout)?; + from_row = next_bounded_primary_key_row( + &mut from_stream, + layouts.from.as_ref(), + &layouts.output_columns, + )?; match identity { RowIdentity::PrimaryKey(key) => Some(RowChange::PrimaryKeyDelete { key, row }), RowIdentity::Rowid(_) => { @@ -1352,7 +1489,11 @@ fn bounded_without_rowid_table( } (None, Some((identity, row))) => { page.rows_scanned = page.rows_scanned.saturating_add(1); - to_row = next_bounded_primary_key_row(&mut to_stream, &layout)?; + to_row = next_bounded_primary_key_row( + &mut to_stream, + layouts.to.as_ref(), + &layouts.output_columns, + )?; match identity { RowIdentity::PrimaryKey(key) => Some(RowChange::PrimaryKeyInsert { key, row }), RowIdentity::Rowid(_) => { @@ -1382,9 +1523,170 @@ fn bounded_without_rowid_table( Ok(page) } +fn bounded_without_rowid_changed_pages( + from_reader: &dyn VolumeRead, + to_reader: &dyn VolumeRead, + from_entry: &MasterEntry, + to_entry: &MasterEntry, + layouts: &WithoutRowidPairLayout, + mode: &BoundedRowDiffMode, +) -> Result { + let from_scanner = TableScanner::new(from_reader).map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to inspect source WITHOUT ROWID pages: {error}" + )) + })?; + let to_scanner = TableScanner::new(to_reader).map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to inspect target WITHOUT ROWID pages: {error}" + )) + })?; + let from_pages = from_scanner + .index_btree_pages(from_entry.root_page) + .map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to enumerate source WITHOUT ROWID pages: {error}" + )) + })? + .into_iter() + .collect::>(); + let to_pages = to_scanner + .index_btree_pages(to_entry.root_page) + .map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to enumerate target WITHOUT ROWID pages: {error}" + )) + })? + .into_iter() + .collect::>(); + let all_pages = from_pages.union(&to_pages).copied().collect::>(); + let mut changed_pages = Vec::new(); + for (index, page_number) in all_pages.into_iter().enumerate() { + if index.is_multiple_of(1_024) { + bounded_cancellation_checkpoint()?; + } + if !from_pages.contains(&page_number) || !to_pages.contains(&page_number) { + changed_pages.push(page_number); + continue; + } + let page_idx = PageIdx::try_new(page_number).ok_or_else(|| { + graft::err::LogicalErr::Other(format!("Invalid WITHOUT ROWID page index {page_number}")) + })?; + let from_page = from_reader.read_page(page_idx)?; + let to_page = to_reader.read_page(page_idx)?; + if from_page != to_page + || from_scanner + .index_page_has_overflow(page_number) + .map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to inspect source WITHOUT ROWID overflow pages: {error}" + )) + })? + || to_scanner + .index_page_has_overflow(page_number) + .map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to inspect target WITHOUT ROWID overflow pages: {error}" + )) + })? + { + changed_pages.push(page_number); + } + } + + let from_rows = changed_without_rowid_records( + &from_scanner, + &from_pages, + &changed_pages, + layouts + .from + .as_ref() + .expect("source WITHOUT ROWID layout exists"), + &layouts.output_columns, + "source", + )?; + let to_rows = changed_without_rowid_records( + &to_scanner, + &to_pages, + &changed_pages, + layouts + .to + .as_ref() + .expect("target WITHOUT ROWID layout exists"), + &layouts.output_columns, + "target", + )?; + let rows_scanned = from_rows.len().saturating_add(to_rows.len()); + let all_changes = diff_sqlite_rows(from_rows, to_rows); + let inserts = all_changes + .iter() + .filter(|change| matches!(change, RowChange::PrimaryKeyInsert { .. })) + .count(); + let deletes = all_changes + .iter() + .filter(|change| matches!(change, RowChange::PrimaryKeyDelete { .. })) + .count(); + let updates = all_changes + .iter() + .filter(|change| matches!(change, RowChange::PrimaryKeyUpdate { .. })) + .count(); + let (changes, has_more, next_offset) = match mode { + BoundedRowDiffMode::Summary => (Vec::new(), false, None), + BoundedRowDiffMode::Rows { offset, limit, .. } => { + let has_more = all_changes.len() > offset.saturating_add(*limit); + let changes = all_changes.into_iter().skip(*offset).take(*limit).collect(); + ( + changes, + has_more, + has_more.then_some(offset.saturating_add(*limit)), + ) + } + }; + Ok(BoundedTablePage { + inserts, + deletes, + updates, + rows_scanned, + primary_key_columns: layouts.primary_key_columns.clone(), + changes, + has_more, + next_offset, + }) +} + +fn changed_without_rowid_records( + scanner: &TableScanner<'_>, + owned_pages: &BTreeSet, + changed_pages: &[u32], + layout: &WithoutRowidLayout, + output_columns: &[String], + side: &str, +) -> Result, graft::err::GraftErr> { + let mut rows = BTreeMap::new(); + for page_number in changed_pages + .iter() + .copied() + .filter(|page_number| owned_pages.contains(page_number)) + { + let records = scanner + .read_index_records_from_page(page_number) + .map_err(|error| { + graft::err::LogicalErr::Other(format!( + "Failed to decode {side} WITHOUT ROWID page {page_number}: {error}" + )) + })?; + for record in records { + let (identity, record) = layout.decode(record, output_columns)?; + rows.insert(identity, record); + } + } + Ok(rows) +} + fn next_bounded_primary_key_row( stream: &mut Option>, - layout: &WithoutRowidLayout, + layout: Option<&WithoutRowidLayout>, + output_columns: &[String], ) -> Result, graft::err::GraftErr> { stream .as_mut() @@ -1396,7 +1698,15 @@ fn next_bounded_primary_key_row( )) })? .flatten() - .map(|record| layout.decode(record)) + .map(|record| { + layout + .ok_or_else(|| { + graft::err::LogicalErr::Other( + "WITHOUT ROWID stream is missing its schema layout".into(), + ) + })? + .decode(record, output_columns) + }) .transpose() } @@ -1407,31 +1717,33 @@ fn bounded_rowid_table( to_entry: Option<&MasterEntry>, mode: &BoundedRowDiffMode, ) -> Result { - let from_count = from_entry - .map(|entry| TableScanner::new(from_reader)?.count_table_rows(entry.root_page)) - .transpose() - .map_err(|error| { - graft::err::LogicalErr::Other(format!("Failed to count source rows: {error}")) - })? - .unwrap_or(0); - let to_count = to_entry - .map(|entry| TableScanner::new(to_reader)?.count_table_rows(entry.root_page)) - .transpose() - .map_err(|error| { - graft::err::LogicalErr::Other(format!("Failed to count target rows: {error}")) - })? - .unwrap_or(0); - if matches!(mode, BoundedRowDiffMode::Summary) && (from_count == 0 || to_count == 0) { - return Ok(BoundedTablePage { - inserts: to_count, - deletes: from_count, - updates: 0, - rows_scanned: 0, - primary_key_columns: Vec::new(), - changes: Vec::new(), - has_more: false, - next_offset: None, - }); + if matches!(mode, BoundedRowDiffMode::Summary) { + let from_count = from_entry + .map(|entry| TableScanner::new(from_reader)?.count_table_rows(entry.root_page)) + .transpose() + .map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to count source rows: {error}")) + })? + .unwrap_or(0); + let to_count = to_entry + .map(|entry| TableScanner::new(to_reader)?.count_table_rows(entry.root_page)) + .transpose() + .map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to count target rows: {error}")) + })? + .unwrap_or(0); + if from_count == 0 || to_count == 0 { + return Ok(BoundedTablePage { + inserts: to_count, + deletes: from_count, + updates: 0, + rows_scanned: 0, + primary_key_columns: Vec::new(), + changes: Vec::new(), + has_more: false, + next_offset: None, + }); + } } let mut from_stream = from_entry .map(|entry| TableScanner::new(from_reader)?.into_row_stream(entry.root_page)) @@ -1692,7 +2004,7 @@ impl MaterializedSnapshot { pub(crate) fn from_reader( reader: &dyn VolumeRead, label: &str, - ) -> Result { + ) -> Result { let directory = tempfile::tempdir().map_err(|error| { graft::err::LogicalErr::Other(format!( "Failed to create temporary {label} SQLite snapshot: {error}" @@ -1705,6 +2017,9 @@ impl MaterializedSnapshot { )) })?; for page_number in 1..=reader.page_count().to_u32() { + if page_number.is_multiple_of(1_024) { + bounded_cancellation_checkpoint()?; + } let page_idx = PageIdx::try_new(page_number).ok_or_else(|| { graft::err::LogicalErr::Other(format!( "Invalid page {page_number} while materializing {label} SQLite snapshot" @@ -1753,7 +2068,7 @@ impl MaterializedPair { fn new( from_reader: &dyn VolumeRead, to_reader: &dyn VolumeRead, - ) -> Result { + ) -> Result { Ok(Self { from: MaterializedSnapshot::from_reader(from_reader, "source")?, to: MaterializedSnapshot::from_reader(to_reader, "target")?, diff --git a/crates/graft-sqlite/src/sqlite_parse.rs b/crates/graft-sqlite/src/sqlite_parse.rs index 107c0b64..41997914 100644 --- a/crates/graft-sqlite/src/sqlite_parse.rs +++ b/crates/graft-sqlite/src/sqlite_parse.rs @@ -390,6 +390,122 @@ impl<'a> TableScanner<'a> { Ok(rows) } + /// Return every page that belongs to an index B-tree. + /// + /// `WITHOUT ROWID` tables use this layout for their primary storage. Keeping page identity + /// lets callers skip byte-identical subtrees and decode only pages that can contain changed + /// rows. + pub fn index_btree_pages(&self, root_page: u32) -> Result, ParseError> { + let mut pages = Vec::new(); + self.collect_index_btree_pages(root_page, &mut pages)?; + Ok(pages) + } + + /// Decode all records stored directly on one index B-tree page. + /// + /// Interior index cells contain complete records as well as child pointers, so they must be + /// included alongside leaf cells when a changed page is inspected. + pub fn read_index_records_from_page(&self, page_num: u32) -> Result, ParseError> { + let (full_page, header_offset, header) = self.read_table_page(page_num)?; + if !matches!(header.page_type, 2 | 10) { + return Err(ParseError::InvalidPage); + } + let page_bytes = full_page.as_ref(); + let interior = header.page_type == 2; + let mut records = Vec::with_capacity(usize::from(header.num_cells)); + for cell_index in 0..header.num_cells { + let ptr_offset = header_offset + header.header_size() + usize::from(cell_index) * 2; + if ptr_offset + 2 > page_bytes.len() { + return Err(ParseError::InvalidCell); + } + let cell_offset = usize::from(u16::from_be_bytes([ + page_bytes[ptr_offset], + page_bytes[ptr_offset + 1], + ])); + records.push(self.read_index_record_from_bytes(page_bytes, cell_offset, interior)?); + } + Ok(records) + } + + /// Return true when an index page references overflow payload pages. + /// + /// The owning B-tree page can remain byte-identical while an overflow page changes. Callers + /// therefore conservatively decode such pages instead of treating equal page bytes as proof + /// that every contained row is unchanged. + pub fn index_page_has_overflow(&self, page_num: u32) -> Result { + let (full_page, header_offset, header) = self.read_table_page(page_num)?; + if !matches!(header.page_type, 2 | 10) { + return Err(ParseError::InvalidPage); + } + let page_bytes = full_page.as_ref(); + let payload_prefix = if header.page_type == 2 { 4 } else { 0 }; + let usable_end = self.usable_size().min(page_bytes.len()); + for cell_index in 0..header.num_cells { + let ptr_offset = header_offset + header.header_size() + usize::from(cell_index) * 2; + if ptr_offset + 2 > page_bytes.len() { + return Err(ParseError::InvalidCell); + } + let cell_offset = usize::from(u16::from_be_bytes([ + page_bytes[ptr_offset], + page_bytes[ptr_offset + 1], + ])); + let payload_offset = cell_offset + .checked_add(payload_prefix) + .ok_or(ParseError::InvalidCell)?; + if payload_offset >= usable_end { + return Err(ParseError::InvalidCell); + } + let (payload_size, bytes_read) = read_varint(&page_bytes[payload_offset..usable_end]); + if payload_size < 0 || bytes_read == 0 { + return Err(ParseError::InvalidCell); + } + let payload_size = + usize::try_from(payload_size).map_err(|_| ParseError::InvalidCell)?; + if self.local_index_payload_size(payload_size) < payload_size { + return Ok(true); + } + } + Ok(false) + } + + fn collect_index_btree_pages( + &self, + page_num: u32, + pages: &mut Vec, + ) -> Result<(), ParseError> { + let (full_page, header_offset, header) = self.read_table_page(page_num)?; + pages.push(page_num); + if header.page_type == 10 { + return Ok(()); + } + if header.page_type != 2 { + return Err(ParseError::InvalidPage); + } + let page_bytes = full_page.as_ref(); + for cell_index in 0..header.num_cells { + let ptr_offset = header_offset + header.header_size() + usize::from(cell_index) * 2; + if ptr_offset + 2 > page_bytes.len() { + return Err(ParseError::InvalidCell); + } + let cell_offset = usize::from(u16::from_be_bytes([ + page_bytes[ptr_offset], + page_bytes[ptr_offset + 1], + ])); + if cell_offset + 4 > page_bytes.len() { + return Err(ParseError::InvalidCell); + } + let left_child = u32::from_be_bytes([ + page_bytes[cell_offset], + page_bytes[cell_offset + 1], + page_bytes[cell_offset + 2], + page_bytes[cell_offset + 3], + ]); + self.collect_index_btree_pages(left_child, pages)?; + } + let right_child = header.right_child_ptr.ok_or(ParseError::InvalidPage)?; + self.collect_index_btree_pages(right_child, pages) + } + fn collect_index_stream_items( &self, page_num: u32, From 4021ca14fdd162a7cf3b808916ab47f0f4ef386f Mon Sep 17 00:00:00 2001 From: Mayne Date: Sun, 2 Aug 2026 14:47:45 +0800 Subject: [PATCH 3/4] chore(release): prepare SDK 0.3.5 and Remote 0.2.1 --- CHANGELOG.md | 50 + Cargo.lock | 5 +- crates/graft-sdk-node/Cargo.toml | 4 +- crates/graft-sdk/Cargo.toml | 2 +- crates/graft-sdk/src/lib.rs | 172 ++- crates/graft-sqlite/Cargo.toml | 1 + crates/graft-sqlite/src/file/vol_file.rs | 28 +- crates/graft-sqlite/src/pragma/repo_diff.rs | 208 ++- .../graft-sqlite/src/pragma/repo_history.rs | 3 +- crates/graft-sqlite/src/pragma/repo_output.rs | 179 +-- .../graft-sqlite/src/pragma/repo_snapshot.rs | 137 +- .../graft-sqlite/src/pragma/repo_staging.rs | 14 +- .../src/pragma/sqlite_worktree.rs | 1100 +++++++++++++++- crates/graft-sqlite/src/row_level_diff.rs | 66 + crates/graft/src/core/commit_hash.rs | 3 +- crates/graft/src/local/fjall_storage.rs | 103 +- crates/graft/src/repo.rs | 6 +- crates/graft/src/repo/inventory.rs | 29 + .../graft/src/rt/action/hydrate_snapshot.rs | 7 +- crates/graft/src/rt/runtime.rs | 98 +- packages/graft-remote-cloudflare/package.json | 2 +- packages/graft-remote-hono/package.json | 2 +- packages/graft-remote/package.json | 2 +- packages/graft-sdk/benchmark/PERFORMANCE.md | 280 ++++ .../benchmark/performance-matrix.mjs | 448 +++++++ packages/graft-sdk/benchmark/real-eidos.mjs | 137 ++ .../results/git-workflow-candidate.json | 454 +++++++ .../results/git-workflow-comparison.md | 44 + .../results/git-workflow-v0.3.4.json | 454 +++++++ ...formance-matrix-candidate-macos-arm64.json | 1135 +++++++++++++++++ ...performance-matrix-v0.3.4-macos-arm64.json | 1135 +++++++++++++++++ .../real-eidos-candidate-macos-arm64.json | 74 ++ .../real-eidos-v0.3.4-macos-arm64.json | 74 ++ packages/graft-sdk/package.json | 4 +- .../graft-sdk/test/repository-session.test.js | 4 +- scripts/remote-release.test.mjs | 4 +- 36 files changed, 6195 insertions(+), 273 deletions(-) create mode 100644 packages/graft-sdk/benchmark/PERFORMANCE.md create mode 100644 packages/graft-sdk/benchmark/performance-matrix.mjs create mode 100644 packages/graft-sdk/benchmark/real-eidos.mjs create mode 100644 packages/graft-sdk/benchmark/results/git-workflow-candidate.json create mode 100644 packages/graft-sdk/benchmark/results/git-workflow-comparison.md create mode 100644 packages/graft-sdk/benchmark/results/git-workflow-v0.3.4.json create mode 100644 packages/graft-sdk/benchmark/results/performance-matrix-candidate-macos-arm64.json create mode 100644 packages/graft-sdk/benchmark/results/performance-matrix-v0.3.4-macos-arm64.json create mode 100644 packages/graft-sdk/benchmark/results/real-eidos-candidate-macos-arm64.json create mode 100644 packages/graft-sdk/benchmark/results/real-eidos-v0.3.4-macos-arm64.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a8ed7789..aed4f5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## Graft SDK 0.3.5 — 2026-08-02 + +### Added + +- Added checksummed, content-addressed SQLite page indexes and worktree probes for large database + files, with conservative fallback when fingerprints are racy or cache data is unavailable. +- Added stable physical SQLite readers, prepared stage state, and snapshot hydration proofs that + stay outside the authoritative repository database write set. +- Added reproducible ordinary-file, Git-like, synthetic SQLite, and real Eidos benchmark suites + with checked-in baseline/candidate raw results and a full performance report. + +### Changed + +- Checkpoint summaries now choose a changed-page-aware rowid algorithm or a bounded primary-key + algorithm based on SQLite table layout instead of applying one scan strategy to every table. +- Staging retains the exact canonical SQLite snapshot and changed-table candidates for commit, + avoiding a second read of the live application database. +- Status refresh preserves proven local classification across Remote projection changes and + refreshes ahead/behind metadata independently. + +### Performance + +- On the checked 460,689,408-byte Eidos fixture, dirty status fell from 5.40 seconds to 19.7 ms, + selected metadata rows from 10.06 seconds to 1.69 ms, and a metadata-only checkpoint commit from + 21.30 seconds to 1.69 ms; peak RSS fell from 3.88 GiB to 672.7 MiB. +- A paired 5.6 MiB Git-like repository workload remained within benchmark noise for ordinary + stage, commit, row diff, checkout, and filesystem Remote push, with unchanged storage bytes. + +### Compatibility + +- SQLite files smaller than 16 MiB bypass the persistent page index because an authoritative scan + is cheaper at that scale. Derived caches are optional and may be deleted without affecting + repository correctness. +- `stagePaths` remains API-compatible but still performs per-path core work; large explicit path + batches remain a documented performance limit. + +## Graft Remote 0.2.1 — 2026-08-02 + +### Added + +- Added resumable multipart upload negotiation for large immutable Remote objects and Cloudflare + R2 multipart storage support. +- Added retry-safe part upload, completion reconciliation, protocol tests, and fallback for Remotes + that do not advertise multipart capabilities. + +### Changed + +- Large segment publication can resume completed parts instead of restarting the entire object + after a timeout or interrupted request. + ## Graft SDK 0.3.4 — 2026-08-01 ### Added diff --git a/Cargo.lock b/Cargo.lock index 48db3c90..fc3eb06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1100,7 +1100,7 @@ dependencies = [ [[package]] name = "graft-sdk" -version = "0.3.4" +version = "0.3.5" dependencies = [ "blake3", "graft", @@ -1115,7 +1115,7 @@ dependencies = [ [[package]] name = "graft-sdk-node" -version = "0.3.4" +version = "0.3.5" dependencies = [ "graft-sdk", "napi", @@ -1128,6 +1128,7 @@ dependencies = [ name = "graft-sqlite" version = "0.11.0" dependencies = [ + "blake3", "bytes", "enum_dispatch", "graft", diff --git a/crates/graft-sdk-node/Cargo.toml b/crates/graft-sdk-node/Cargo.toml index e3c37f62..befc4af4 100644 --- a/crates/graft-sdk-node/Cargo.toml +++ b/crates/graft-sdk-node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "graft-sdk-node" -version = "0.3.4" +version = "0.3.5" edition = "2024" authors.workspace = true license.workspace = true @@ -18,7 +18,7 @@ crate-type = ["cdylib"] workspace = true [dependencies] -graft-sdk = { path = "../graft-sdk", version = "0.3.4", features = ["bundled-sqlite"] } +graft-sdk = { path = "../graft-sdk", version = "0.3.5", features = ["bundled-sqlite"] } napi = { workspace = true } napi-derive = { workspace = true } serde_json = "1.0" diff --git a/crates/graft-sdk/Cargo.toml b/crates/graft-sdk/Cargo.toml index 688e5f31..241fdade 100644 --- a/crates/graft-sdk/Cargo.toml +++ b/crates/graft-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "graft-sdk" -version = "0.3.4" +version = "0.3.5" edition = "2024" authors.workspace = true license.workspace = true diff --git a/crates/graft-sdk/src/lib.rs b/crates/graft-sdk/src/lib.rs index 9975f29c..8125bb19 100644 --- a/crates/graft-sdk/src/lib.rs +++ b/crates/graft-sdk/src/lib.rs @@ -41,7 +41,7 @@ const MAX_BATCH_MUTATION_PATHS: usize = 1_000; const MAX_INVENTORY_PAGE_SIZE: usize = 1_000; const MAX_IGNORE_QUERY_PATHS: usize = 1_000; // Bump whenever persisted path classification semantics change. -const STATUS_SNAPSHOT_SCHEMA_VERSION: u32 = 2; +const STATUS_SNAPSHOT_SCHEMA_VERSION: u32 = 3; const MAX_STATUS_SNAPSHOTS: usize = 4; const STATUS_SNAPSHOT_MAX_BYTES: u64 = 256 * 1024 * 1024; const WORKTREE_STABILITY_ATTEMPTS: usize = 3; @@ -1428,7 +1428,7 @@ impl RepositorySession { self.set_http_bearer_token(&options.name, token.clone())?; } - let result = self.with_service(|service| { + self.with_service(|service| { let remotes = execute_json(service, "json_remotes", None)?; let existing_url = remote_url(&remotes, &options.name)?; match existing_url { @@ -1458,11 +1458,7 @@ impl RepositorySession { execute_json(service, "json_branch_upstream", Some(&argument))?; } execute_json(service, "json_remotes", None) - }); - if result.is_ok() { - let _ = self.invalidate_status_cache(); - } - result + }) } pub fn push(&self, remote: Option<&str>, branch: Option<&str>) -> Result { @@ -1472,7 +1468,9 @@ impl RepositorySession { pub fn fetch(&self, remote: Option<&str>, branch: Option<&str>) -> Result { let argument = remote_branch_argument(remote, branch)?; - self.execute_json_mutating("json_fetch", argument.as_deref()) + // Fetch updates objects and remote-tracking refs, not the local worktree. Preserve the + // proven local classification and refresh only the repository projection on next status. + self.execute_json("json_fetch", argument.as_deref()) } pub fn pull(&self, remote: Option<&str>, branch: Option<&str>) -> Result { @@ -1514,13 +1512,6 @@ impl RepositorySession { }) } - fn invalidate_status_cache(&self) -> Result<()> { - self.with_state(|state| { - state.status_cache.invalidate(); - Ok(()) - }) - } - fn with_service( &self, operation: impl FnOnce(&mut RepositoryCommandService) -> Result, @@ -1659,10 +1650,27 @@ fn refresh_incremental_status_once( + matching_fingerprint_count(&cache.untracked_fingerprints, &untracked); let paths_examined = tracked.len() + untracked.len(); if tracked == cache.tracked_fingerprints && untracked == cache.untracked_fingerprints { - let status = cache + let previous_status = cache .status .clone() .expect("initialized status cache contains a status"); + let mut status = previous_status.clone(); + repo.refresh_status_repository_projection(&mut status) + .map_err(repo_error)?; + let projection_changed = status_changed(Some(&previous_status), &status)?; + if projection_changed { + cache.generation = cache.generation.saturating_add(1).max(1); + cache.status = Some(status.clone()); + } + let persistent_snapshot_saved = if projection_changed { + match persist_status_snapshot(&repo, cache) { + Ok(saved) => saved, + Err(error) if error.code() == SdkErrorCode::Cancelled => return Err(error), + Err(_) => false, + } + } else { + false + }; return Ok(incremental_status_result( cache, status, @@ -1675,7 +1683,7 @@ fn refresh_incremental_status_once( tree_cache_hit, status_cache_hit: true, persistent_snapshot_hit, - persistent_snapshot_saved: false, + persistent_snapshot_saved, stability_retries: 0, }, )); @@ -1753,9 +1761,10 @@ fn incremental_status_result( ) -> IncrementalStatusResult { telemetry.duration_us = elapsed_us(started); let head = cache.head_target.as_deref().unwrap_or("unborn"); - let status_digest = serde_json::to_vec(&status) - .map(|bytes| blake3::hash(&bytes).to_hex().to_string()) - .unwrap_or_else(|_| "unavailable".to_string()); + let status_digest = serde_json::to_vec(&status).map_or_else( + |_| "unavailable".to_string(), + |bytes| blake3::hash(&bytes).to_hex().to_string(), + ); IncrementalStatusResult { generation: cache.generation, change_token: format!("{head}:{}:{status_digest}", cache.generation), @@ -2118,17 +2127,17 @@ fn repository_metadata_fingerprint(repo: &Repository, index: &Index) -> Result Result<()> { - let entries = match fs::read_dir(directory) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(repository_stale_io("read repository metadata", error)), - }; - let mut paths = entries - .map(|entry| { - entry - .map(|entry| entry.path()) - .map_err(|error| repository_stale_io("read repository metadata entry", error)) - }) - .collect::>>()?; - paths.sort(); - for path in paths { - graft::repo::cancellation_checkpoint().map_err(repo_error)?; - let metadata = fs::symlink_metadata(&path) - .map_err(|error| repository_stale_io("inspect repository metadata", error))?; - if metadata.is_dir() { - hash_metadata_tree(hasher, base, &path)?; - } else if metadata.is_file() { - hash_optional_file(hasher, base, &path)?; - } - } - Ok(()) -} - fn hash_optional_file(hasher: &mut blake3::Hasher, base: &Path, path: &Path) -> Result<()> { graft::repo::cancellation_checkpoint().map_err(repo_error)?; let before = match fingerprint_path(path)? { @@ -3193,6 +3175,67 @@ mod tests { assert!(!encoded.contains(&directory.path().to_string_lossy().to_string())); } + #[test] + fn remote_tracking_updates_reuse_local_status_proof_and_refresh_projection() { + let directory = tempfile::tempdir().unwrap(); + let note = directory.path().join("note.txt"); + let session = RepositorySession::new(directory.path()); + session.open().unwrap(); + session.init().unwrap(); + + fs::write(¬e, "one\n").unwrap(); + session.add_all().unwrap(); + let first = session.commit("first").unwrap()["commit"]["id"] + .as_str() + .unwrap() + .to_string(); + session + .configure_remote(&RemoteConfigureOptions { + name: "origin".to_string(), + url: "https://example.invalid/acme/repo".to_string(), + bearer_token: None, + overwrite: false, + upstream_branch: Some("main".to_string()), + }) + .unwrap(); + let writer = Repository::open(directory.path()).unwrap(); + writer + .set_remote_tracking_ref("origin", "main", &first) + .unwrap(); + let initial = session.status_incremental().unwrap(); + assert_eq!(initial.status.ahead, 0); + + fs::write(¬e, "two\n").unwrap(); + session.add_all().unwrap(); + let second = session.commit("second").unwrap()["commit"]["id"] + .as_str() + .unwrap() + .to_string(); + let ahead = session.status_incremental().unwrap(); + assert_eq!(ahead.status.ahead, 1); + + writer + .set_remote_tracking_ref("origin", "main", &second) + .unwrap(); + let hot_synced = session.status_incremental().unwrap(); + assert!(hot_synced.telemetry.status_cache_hit); + assert!(hot_synced.telemetry.persistent_snapshot_saved); + assert_eq!(hot_synced.status.ahead, 0); + assert_eq!(hot_synced.status.behind, 0); + assert!(hot_synced.generation > ahead.generation); + + session.close().unwrap(); + writer + .set_remote_tracking_ref("origin", "main", &first) + .unwrap(); + session.open().unwrap(); + let reopened_ahead = session.status_incremental().unwrap(); + assert!(reopened_ahead.telemetry.persistent_snapshot_hit); + assert!(reopened_ahead.telemetry.status_cache_hit); + assert_eq!(reopened_ahead.status.ahead, 1); + assert_eq!(reopened_ahead.status.behind, 0); + } + #[test] fn persistent_status_snapshot_rejects_older_classification_schema() { let directory = tempfile::tempdir().unwrap(); @@ -3588,6 +3631,17 @@ mod tests { .unwrap(); drop(database); + // Requesting one table first must not require a whole-file worktree probe. + let rows = session + .diff_sqlite_paths(&options(SqliteDiffResponse::Rows { + table: "archive".to_string(), + limit: 100, + after: None, + })) + .unwrap(); + assert_eq!(rows.telemetry.rows_returned, 3); + assert!(rows.telemetry.rows_scanned < 2_000); + let summary = session .diff_sqlite_paths(&options(SqliteDiffResponse::Summary)) .unwrap(); @@ -3599,16 +3653,6 @@ mod tests { assert_eq!(file["summaries"][0]["updates"], 1); assert!(summary.telemetry.rows_scanned < 2_000); - let rows = session - .diff_sqlite_paths(&options(SqliteDiffResponse::Rows { - table: "archive".to_string(), - limit: 100, - after: None, - })) - .unwrap(); - assert_eq!(rows.telemetry.rows_returned, 3); - assert!(rows.telemetry.rows_scanned < 2_000); - session .stage_paths(&StagePathsOptions { paths: vec![PathBuf::from("space.eidos")], diff --git a/crates/graft-sqlite/Cargo.toml b/crates/graft-sqlite/Cargo.toml index 580ba0bc..3111eb36 100644 --- a/crates/graft-sqlite/Cargo.toml +++ b/crates/graft-sqlite/Cargo.toml @@ -18,6 +18,7 @@ graft = { path = "../graft", version = "0.11.0" } serde = { workspace = true, features = ["derive"] } serde_json = "1.0" +blake3 = { workspace = true } bytes = { workspace = true } enum_dispatch = { workspace = true } indoc = { workspace = true } diff --git a/crates/graft-sqlite/src/file/vol_file.rs b/crates/graft-sqlite/src/file/vol_file.rs index 3e059333..d6c54ddd 100644 --- a/crates/graft-sqlite/src/file/vol_file.rs +++ b/crates/graft-sqlite/src/file/vol_file.rs @@ -1,5 +1,6 @@ use std::{ borrow::Cow, + collections::BTreeMap, fmt::Debug, hash::{DefaultHasher, Hash, Hasher}, mem, @@ -27,7 +28,10 @@ use graft::{ use parking_lot::{Mutex, MutexGuard}; use sqlite_plugin::flags::{LockLevel, OpenOpts}; -use crate::vfs::{ErrCtx, RepoRuntimeRegistry}; +use crate::{ + pragma::sqlite_worktree::PreparedSqliteStage, + vfs::{ErrCtx, RepoRuntimeRegistry}, +}; use super::VfsFile; @@ -134,6 +138,13 @@ pub struct VolFile { state: VolFileState, /// Message to attach to the next commit (consumed on commit). pub pending_message: Option, + /// Stable `SQLite` snapshots prepared by `add` and reusable by the following commit. + /// + /// This is the repository equivalent of Git's index cache: commit consumes the exact staged + /// image instead of proving it again by rereading the live worktree. Entries are still matched + /// against the staged index before use, so stale cache entries only cause a conservative + /// fallback. + prepared_sqlite_stages: Mutex>>, } impl Debug for VolFile { @@ -247,6 +258,7 @@ impl VolFile { reserved, state: VolFileState::Idle, pending_message: None, + prepared_sqlite_stages: Mutex::new(BTreeMap::new()), } } @@ -254,6 +266,20 @@ impl VolFile { &self.runtime } + pub(crate) fn cache_prepared_sqlite_stage(&self, key: String, prepared: PreparedSqliteStage) { + self.prepared_sqlite_stages + .lock() + .insert(key, Arc::new(prepared)); + } + + pub(crate) fn prepared_sqlite_stage(&self, key: &str) -> Option> { + self.prepared_sqlite_stages.lock().get(key).cloned() + } + + pub(crate) fn clear_prepared_sqlite_stages(&self) { + self.prepared_sqlite_stages.lock().clear(); + } + /// Returns the `SQLite` database selected for row-aware repository operations. /// /// VFS-backed files use their open database path. A control-plane-only repository session has diff --git a/crates/graft-sqlite/src/pragma/repo_diff.rs b/crates/graft-sqlite/src/pragma/repo_diff.rs index f5957575..f894245b 100644 --- a/crates/graft-sqlite/src/pragma/repo_diff.rs +++ b/crates/graft-sqlite/src/pragma/repo_diff.rs @@ -7,6 +7,8 @@ pub(super) fn repo_diff_for_spec( spec: RepoDiffSpec, ) -> Result { let kind = spec.kind; + let bounded_table_rows = + spec.mode == DiffMode::Rows && spec.table.is_some() && spec.row_page.is_some(); let mut diff = match spec.target { RepoDiffTarget::Worktree { path } => { let path = repo_diff_path(repo, path.as_deref())?; @@ -26,14 +28,24 @@ pub(super) fn repo_diff_for_spec( )), Ok(_) if is_sqlite_database_path(&physical_path)? => { let expected = repo.index_file(&physical_path)?; - repo_diff_physical_sqlite_file( - runtime, - repo, - &physical_path, - &key, - expected, - None, - ) + if bounded_table_rows { + repo_diff_physical_sqlite_file_for_bounded_rows( + repo, + &physical_path, + &key, + expected, + None, + ) + } else { + repo_diff_physical_sqlite_file( + runtime, + repo, + &physical_path, + &key, + expected, + None, + ) + } } Ok(_) => Ok(repo.diff_worktree_artifact(&physical_path, Some(&key))?), Err(err) if err.kind() == std::io::ErrorKind::NotFound => { @@ -74,14 +86,24 @@ pub(super) fn repo_diff_for_spec( )), Ok(_) if is_sqlite_database_path(&physical_path)? => { let expected = repo.file_from_revision(&rev, &physical_path)?; - repo_diff_physical_sqlite_file( - runtime, - repo, - &physical_path, - &key, - expected, - Some(&rev), - ) + if bounded_table_rows { + repo_diff_physical_sqlite_file_for_bounded_rows( + repo, + &physical_path, + &key, + expected, + Some(&rev), + ) + } else { + repo_diff_physical_sqlite_file( + runtime, + repo, + &physical_path, + &key, + expected, + Some(&rev), + ) + } } Ok(_) => Ok(repo.diff_revision_to_worktree_artifact( &rev, @@ -453,19 +475,24 @@ pub(super) fn repo_diff_physical_sqlite_file( expected: Option, rev: Option<&str>, ) -> Result { - let physical = PhysicalSqliteReader::open(physical_path)?; - let matches = expected - .as_ref() - .map(|expected| physical.matches_state(runtime, expected)) - .transpose()? - .unwrap_or(false); + let (matches, worktree_state) = super::sqlite_worktree::with_consistent_physical_sqlite_reader( + physical_path, + |physical| { + let matches = expected + .as_ref() + .map(|expected| physical.matches_cached_state(runtime, repo, key, expected)) + .transpose()? + .unwrap_or(false); + Ok((matches, physical.worktree_state())) + }, + )?; let state = if matches { expected.expect("matching physical file has an expected state") } else { CommitFileState { volume: VolumeId::EMPTY, snapshot: RepoSnapshot { - page_count: physical.page_count(), + page_count: worktree_state.page_count, ranges: Vec::new(), }, } @@ -482,11 +509,50 @@ pub(super) fn repo_diff_physical_sqlite_file( .find(|file| file.path == key) .expect("changed physical SQLite file should produce a diff entry"); file.to = None; - file.worktree = Some(physical.worktree_state()); + file.worktree = Some(worktree_state); } Ok(diff) } +/// Builds a worktree diff shell for a table-scoped bounded row request. +/// +/// The subsequent row diff compares only the requested table, so probing every page merely to +/// rediscover that the physical file changed would defeat the table fast path. Summary/status +/// requests continue to use the authoritative whole-file probe above. +fn repo_diff_physical_sqlite_file_for_bounded_rows( + repo: &Repository, + physical_path: &Path, + key: &str, + expected: Option, + rev: Option<&str>, +) -> Result { + let worktree_state = super::sqlite_worktree::with_consistent_physical_sqlite_reader( + physical_path, + |physical| Ok(physical.worktree_state()), + )?; + let state = CommitFileState { + volume: VolumeId::EMPTY, + snapshot: RepoSnapshot { + page_count: worktree_state.page_count, + ranges: Vec::new(), + }, + }; + let mut diff = if let Some(rev) = rev { + repo.diff_revision_to_worktree_file(rev, physical_path, state, Some(key))? + } else { + repo.diff_worktree_file(physical_path, state, Some(key))? + }; + let file = diff + .files + .iter_mut() + .find(|file| file.path == key) + .expect("bounded physical SQLite request should produce a diff entry"); + file.from = expected; + file.to = None; + file.worktree = Some(worktree_state); + Ok(diff) +} + pub(super) fn repo_has_work_in_progress_for_file( runtime: &Runtime, file: &VolFile, @@ -576,11 +642,27 @@ pub(super) fn repo_file_state_content_eq( pub(super) fn staged_commit_table_summary( runtime: &Runtime, repo: &Repository, +) -> Result, ErrCtx> { + staged_commit_table_summary_with_prepared(runtime, repo, None) +} + +pub(super) fn staged_commit_table_summary_for_file( + runtime: &Runtime, + file: &VolFile, + repo: &Repository, +) -> Result, ErrCtx> { + staged_commit_table_summary_with_prepared(runtime, repo, Some(file)) +} + +fn staged_commit_table_summary_with_prepared( + runtime: &Runtime, + repo: &Repository, + prepared_file: Option<&VolFile>, ) -> Result, ErrCtx> { let diff = repo.diff_staged(None)?; let mut by_name = BTreeMap::::new(); for file in &diff.files { - let summaries = repo_file_table_summary(runtime, repo, file)?; + let summaries = repo_file_table_summary_with_prepared(runtime, repo, file, prepared_file)?; for summary in summaries { merge_table_summary(&mut by_name, summary); } @@ -588,10 +670,11 @@ pub(super) fn staged_commit_table_summary( Ok(by_name.into_values().collect()) } -pub(super) fn repo_file_table_summary( +fn repo_file_table_summary_with_prepared( runtime: &Runtime, repo: &Repository, file: &graft::repo::RepoFileDiff, + prepared_file: Option<&VolFile>, ) -> Result, ErrCtx> { match (&file.from, &file.to) { (Some(from), Some(to)) => { @@ -611,9 +694,14 @@ pub(super) fn repo_file_table_summary( SnapshotSummaryMode::Deleted, ); } - if let Some(summaries) = - staged_worktree_table_summary(runtime, repo, file, &from_snapshot, &to_snapshot)? - { + if let Some(summaries) = staged_worktree_table_summary( + runtime, + repo, + file, + &from_snapshot, + &to_snapshot, + prepared_file, + )? { return Ok(summaries); } let from_reader = runtime.snapshot_reader(from_snapshot.clone()); @@ -661,26 +749,62 @@ fn staged_worktree_table_summary( file: &graft::repo::RepoFileDiff, from_snapshot: &graft::snapshot::Snapshot, to_snapshot: &graft::snapshot::Snapshot, + prepared_file: Option<&VolFile>, ) -> Result>, ErrCtx> { - let physical_path = repo.worktree().join(&file.path); - let metadata = match std::fs::symlink_metadata(&physical_path) { - Ok(metadata) if metadata.file_type().is_file() => metadata, - Ok(_) => return Ok(None), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), + let prepared = + prepared_file.and_then(|prepared_file| prepared_file.prepared_sqlite_stage(&file.path)); + let tables = match (&file.from, &file.to, prepared.as_ref()) { + (Some(from), Some(to), Some(prepared)) if prepared.matches(from, to) => { + prepared.table_candidates()? + } + _ => { + let physical_path = repo.worktree().join(&file.path); + let metadata = match std::fs::symlink_metadata(&physical_path) { + Ok(metadata) if metadata.file_type().is_file() => metadata, + Ok(_) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if metadata.len() < 100 || !is_sqlite_database_path(&physical_path)? { + return Ok(None); + } + + let physical = PhysicalSqliteReader::open(&physical_path)?; + let from_reader = runtime.snapshot_reader(from_snapshot.clone()); + let to_reader = runtime.snapshot_reader(to_snapshot.clone()); + physical.staged_table_candidates(&to_reader, &from_reader)? + } }; - if metadata.len() < 100 || !is_sqlite_database_path(&physical_path)? { + let Some(tables) = tables else { return Ok(None); - } - - let physical = PhysicalSqliteReader::open(&physical_path)?; + }; let from_reader = runtime.snapshot_reader(from_snapshot.clone()); let to_reader = runtime.snapshot_reader(to_snapshot.clone()); - let Some(tables) = physical.staged_table_candidates(&to_reader, &from_reader)? else { - return Ok(None); - }; let from_lsn = from_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); let to_lsn = to_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + if let Some(summaries) = crate::row_level_diff::rowid_table_summaries_for_tables( + &from_reader, + &to_reader, + from_lsn, + to_lsn, + &tables, + ) + .map_err(|error| ErrCtx::PragmaErr(format!("Diff error: {error:?}").into()))? + { + return Ok(Some( + summaries + .into_iter() + .filter_map(|summary| { + table_summary( + summary.table_name, + summary.inserts, + summary.deletes, + summary.updates, + ) + }) + .collect(), + )); + } let diff = crate::row_level_diff::bounded_row_level_diff_readers_for_summary_tables( &from_reader, &to_reader, diff --git a/crates/graft-sqlite/src/pragma/repo_history.rs b/crates/graft-sqlite/src/pragma/repo_history.rs index 60f33628..b4756272 100644 --- a/crates/graft-sqlite/src/pragma/repo_history.rs +++ b/crates/graft-sqlite/src/pragma/repo_history.rs @@ -14,7 +14,7 @@ pub(super) fn run_repo_commit( if repo.has_configured_track_roots()? { stage_repo_add_all(runtime, file, &repo, None)?; } - let tables = staged_commit_table_summary(runtime, &repo)?; + let tables = staged_commit_table_summary_for_file(runtime, file, &repo)?; let commit = match repo.commit_staged_with_table_summary(message, tables) { Ok(commit) => commit, Err(graft::repo::RepoErr::NoStagedChanges) => { @@ -23,6 +23,7 @@ pub(super) fn run_repo_commit( Err(err) => return Err(err.into()), }; let branch = repo.current_branch()?; + file.clear_prepared_sqlite_stages(); // Staging already captured the canonical SQLite snapshot in the index. Re-materializing that // snapshot here would replace the worktree directory entry and detach application SQLite // handles that were intentionally kept open across a non-materializing checkpoint. diff --git a/crates/graft-sqlite/src/pragma/repo_output.rs b/crates/graft-sqlite/src/pragma/repo_output.rs index 14f5ef53..1c650160 100644 --- a/crates/graft-sqlite/src/pragma/repo_output.rs +++ b/crates/graft-sqlite/src/pragma/repo_output.rs @@ -282,93 +282,118 @@ pub(super) fn repo_file_bounded_row_diff( RepoSnapshotPurpose::Diff, SnapshotHashPolicy::AllowHydratedMismatch, ); - let result = if file.worktree.is_some() { + if file.worktree.is_some() { let physical_path = repo.worktree().join(&file.path); - let physical = PhysicalSqliteReader::open(&physical_path)?; - if let Some(from) = &file.from { + let result = super::sqlite_worktree::with_consistent_physical_sqlite_reader( + &physical_path, + |physical| { + if let Some(from) = &file.from { + resolver.resolve_snapshot(&from.snapshot)?; + let snapshot = from.snapshot.to_snapshot(); + let from_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let from_reader = runtime.snapshot_reader(snapshot); + let summary_tables = + matches!(mode, crate::row_level_diff::BoundedRowDiffMode::Summary) + .then(|| { + physical.cached_changed_table_candidates( + runtime, + repo, + &file.path, + from, + &from_reader, + ) + }) + .transpose()?; + let rows = if let Some(Some(tables)) = summary_tables { + crate::row_level_diff::bounded_row_level_diff_readers_for_summary_tables( + &from_reader, + physical, + from_lsn, + from_lsn.saturating_next(), + &tables, + ) + } else { + crate::row_level_diff::bounded_row_level_diff_readers( + &from_reader, + physical, + from_lsn, + from_lsn.saturating_next(), + mode, + ) + }; + rows.map(Some).map_err(|error| { + ErrCtx::PragmaErr( + format!("Bounded row diff error for `{}`: {error:?}", file.path).into(), + ) + }) + } else { + let empty = empty_sqlite_reader()?; + crate::row_level_diff::bounded_row_level_diff_readers( + &empty, + physical, + LSN::FIRST, + LSN::FIRST.saturating_next(), + mode, + ) + .map(Some) + .map_err(|error| { + ErrCtx::PragmaErr( + format!("Bounded row diff error for `{}`: {error:?}", file.path).into(), + ) + }) + } + }, + ); + return result; + } + + let result = match (&file.from, &file.to) { + (Some(from), Some(to)) => { resolver.resolve_snapshot(&from.snapshot)?; - let snapshot = from.snapshot.to_snapshot(); - let from_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); - let from_reader = runtime.snapshot_reader(snapshot); - let summary_tables = matches!(mode, crate::row_level_diff::BoundedRowDiffMode::Summary) - .then(|| physical.changed_table_candidates(&from_reader)) - .transpose()?; - if let Some(Some(tables)) = summary_tables { - crate::row_level_diff::bounded_row_level_diff_readers_for_summary_tables( - &from_reader, - &physical, - from_lsn, - from_lsn.saturating_next(), - &tables, - ) - } else { - crate::row_level_diff::bounded_row_level_diff_readers( - &from_reader, - &physical, - from_lsn, - from_lsn.saturating_next(), - mode, - ) - } - } else { + resolver.resolve_snapshot(&to.snapshot)?; + let from_snapshot = from.snapshot.to_snapshot(); + let to_snapshot = to.snapshot.to_snapshot(); + let from_lsn = from_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let to_lsn = to_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let from_reader = runtime.snapshot_reader(from_snapshot); + let to_reader = runtime.snapshot_reader(to_snapshot); + crate::row_level_diff::bounded_row_level_diff_readers( + &from_reader, + &to_reader, + from_lsn, + to_lsn, + mode, + ) + } + (None, Some(to)) => { + resolver.resolve_snapshot(&to.snapshot)?; let empty = empty_sqlite_reader()?; + let snapshot = to.snapshot.to_snapshot(); + let to_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let reader = runtime.snapshot_reader(snapshot); crate::row_level_diff::bounded_row_level_diff_readers( &empty, - &physical, + &reader, LSN::FIRST, - LSN::FIRST.saturating_next(), + to_lsn, mode, ) } - } else { - match (&file.from, &file.to) { - (Some(from), Some(to)) => { - resolver.resolve_snapshot(&from.snapshot)?; - resolver.resolve_snapshot(&to.snapshot)?; - let from_snapshot = from.snapshot.to_snapshot(); - let to_snapshot = to.snapshot.to_snapshot(); - let from_lsn = from_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); - let to_lsn = to_snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); - let from_reader = runtime.snapshot_reader(from_snapshot); - let to_reader = runtime.snapshot_reader(to_snapshot); - crate::row_level_diff::bounded_row_level_diff_readers( - &from_reader, - &to_reader, - from_lsn, - to_lsn, - mode, - ) - } - (None, Some(to)) => { - resolver.resolve_snapshot(&to.snapshot)?; - let empty = empty_sqlite_reader()?; - let snapshot = to.snapshot.to_snapshot(); - let to_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); - let reader = runtime.snapshot_reader(snapshot); - crate::row_level_diff::bounded_row_level_diff_readers( - &empty, - &reader, - LSN::FIRST, - to_lsn, - mode, - ) - } - (Some(from), None) => { - resolver.resolve_snapshot(&from.snapshot)?; - let empty = empty_sqlite_reader()?; - let snapshot = from.snapshot.to_snapshot(); - let from_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); - let reader = runtime.snapshot_reader(snapshot); - crate::row_level_diff::bounded_row_level_diff_readers( - &reader, - &empty, - from_lsn, - from_lsn.saturating_next(), - mode, - ) - } - (None, None) => return Ok(None), + (Some(from), None) => { + resolver.resolve_snapshot(&from.snapshot)?; + let empty = empty_sqlite_reader()?; + let snapshot = from.snapshot.to_snapshot(); + let from_lsn = snapshot.head().map_or(LSN::FIRST, |(_, lsn)| lsn); + let reader = runtime.snapshot_reader(snapshot); + crate::row_level_diff::bounded_row_level_diff_readers( + &reader, + &empty, + from_lsn, + from_lsn.saturating_next(), + mode, + ) } + (None, None) => return Ok(None), }; result.map(Some).map_err(|error| { ErrCtx::PragmaErr(format!("Bounded row diff error for `{}`: {error:?}", file.path).into()) diff --git a/crates/graft-sqlite/src/pragma/repo_snapshot.rs b/crates/graft-sqlite/src/pragma/repo_snapshot.rs index dce5896c..722a29f9 100644 --- a/crates/graft-sqlite/src/pragma/repo_snapshot.rs +++ b/crates/graft-sqlite/src/pragma/repo_snapshot.rs @@ -37,7 +37,7 @@ pub(super) struct RepoSnapshotResolvePolicy { pub(super) normalize: bool, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub(super) struct ResolvedRepoSnapshot { pub(super) snapshot: RepoSnapshot, pub(super) runtime_snapshot: graft::snapshot::Snapshot, @@ -51,6 +51,65 @@ pub(super) struct RepoSnapshotResolver<'a> { pub(super) policy: RepoSnapshotResolvePolicy, } +const DIFF_SNAPSHOT_CACHE_CAPACITY: usize = 32; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct DiffSnapshotCacheKey { + runtime_instance_id: u64, + snapshot: RepoSnapshot, +} + +#[derive(Clone, Debug)] +struct DiffSnapshotCacheEntry { + key: DiffSnapshotCacheKey, + resolved: ResolvedRepoSnapshot, +} + +static DIFF_SNAPSHOT_CACHE: std::sync::OnceLock< + parking_lot::Mutex>, +> = std::sync::OnceLock::new(); + +fn cached_diff_snapshot( + runtime: &Runtime, + snapshot: &RepoSnapshot, +) -> Option { + let key = DiffSnapshotCacheKey { + runtime_instance_id: runtime.instance_id(), + snapshot: snapshot.clone(), + }; + let mut cache = DIFF_SNAPSHOT_CACHE + .get_or_init(|| parking_lot::Mutex::new(std::collections::VecDeque::new())) + .lock(); + let index = cache.iter().position(|entry| entry.key == key)?; + let entry = cache + .remove(index) + .expect("diff snapshot cache entry exists"); + let resolved = entry.resolved.clone(); + cache.push_back(entry); + Some(resolved) +} + +fn cache_diff_snapshot( + runtime: &Runtime, + snapshot: &RepoSnapshot, + resolved: &ResolvedRepoSnapshot, +) { + let key = DiffSnapshotCacheKey { + runtime_instance_id: runtime.instance_id(), + snapshot: snapshot.clone(), + }; + let mut cache = DIFF_SNAPSHOT_CACHE + .get_or_init(|| parking_lot::Mutex::new(std::collections::VecDeque::new())) + .lock(); + if let Some(index) = cache.iter().position(|entry| entry.key == key) { + cache.remove(index); + } + cache.push_back(DiffSnapshotCacheEntry { key, resolved: resolved.clone() }); + while cache.len() > DIFF_SNAPSHOT_CACHE_CAPACITY { + cache.pop_front(); + } +} + pub(super) fn hydrate_repo_file_state( runtime: &Runtime, state: &CommitFileState, @@ -237,23 +296,31 @@ impl<'a> RepoSnapshotResolver<'a> { &self, snapshot: &RepoSnapshot, ) -> Result { - if self.policy.remote_mode == RepoSnapshotRemoteMode::Remote { - return self.resolve_snapshot_once( + let cacheable_diff = self.policy.purpose == RepoSnapshotPurpose::Diff + && self.policy.hash_policy == SnapshotHashPolicy::AllowHydratedMismatch + && !self.policy.normalize; + if cacheable_diff { + if let Some(resolved) = cached_diff_snapshot(self.runtime, snapshot) { + return Ok(resolved); + } + } + + let resolved = if self.policy.remote_mode == RepoSnapshotRemoteMode::Remote { + self.resolve_snapshot_once( snapshot, RepoSnapshotResolveSource::Remote, self.remote.clone(), - ); - } - - match self.resolve_snapshot_once(snapshot, RepoSnapshotResolveSource::Local, None) { - Ok(resolved) => Ok(resolved), - Err(local_err) - if self.policy.remote_mode == RepoSnapshotRemoteMode::LocalThenRemote => - { - let Some(remote) = self.remote.clone() else { - return Err(local_err); - }; - self.resolve_snapshot_once(snapshot, RepoSnapshotResolveSource::Remote, Some(remote)) + )? + } else { + match self.resolve_snapshot_once(snapshot, RepoSnapshotResolveSource::Local, None) { + Ok(resolved) => Ok(resolved), + Err(local_err) + if self.policy.remote_mode == RepoSnapshotRemoteMode::LocalThenRemote => + { + let Some(remote) = self.remote.clone() else { + return Err(local_err); + }; + self.resolve_snapshot_once(snapshot, RepoSnapshotResolveSource::Remote, Some(remote)) .map_err(|remote_err| { ErrCtx::PragmaErr( format!( @@ -262,9 +329,14 @@ impl<'a> RepoSnapshotResolver<'a> { .into(), ) }) - } - Err(err) => Err(err), + } + Err(err) => Err(err), + }? + }; + if cacheable_diff { + cache_diff_snapshot(self.runtime, snapshot, &resolved); } + Ok(resolved) } pub(super) fn resolve_snapshot_once( @@ -275,21 +347,24 @@ impl<'a> RepoSnapshotResolver<'a> { ) -> Result { let runtime_snapshot = snapshot.to_snapshot(); if !runtime_snapshot.is_empty() { - match source { - RepoSnapshotResolveSource::Local => { - for range in &snapshot.ranges { - self.runtime.fetch_log(range.log.clone(), Some(range.end))?; + let hydration_cached = self.runtime.snapshot_hydration_cached(&runtime_snapshot)?; + if !hydration_cached { + match source { + RepoSnapshotResolveSource::Local => { + for range in &snapshot.ranges { + self.runtime.fetch_log(range.log.clone(), Some(range.end))?; + } + self.runtime.snapshot_hydrate(runtime_snapshot.clone())?; + } + RepoSnapshotResolveSource::Remote => { + let Some(remote) = remote else { + return Err(ErrCtx::PragmaErr( + "snapshot resolver remote source requires a remote".into(), + )); + }; + self.runtime + .snapshot_hydrate_from(runtime_snapshot.clone(), remote)?; } - self.runtime.snapshot_hydrate(runtime_snapshot.clone())?; - } - RepoSnapshotResolveSource::Remote => { - let Some(remote) = remote else { - return Err(ErrCtx::PragmaErr( - "snapshot resolver remote source requires a remote".into(), - )); - }; - self.runtime - .snapshot_hydrate_from(runtime_snapshot.clone(), remote)?; } } } diff --git a/crates/graft-sqlite/src/pragma/repo_staging.rs b/crates/graft-sqlite/src/pragma/repo_staging.rs index 237a2462..43b729c3 100644 --- a/crates/graft-sqlite/src/pragma/repo_staging.rs +++ b/crates/graft-sqlite/src/pragma/repo_staging.rs @@ -621,7 +621,19 @@ pub(super) fn prepare_repo_add_file( .get(key) .cloned() .or(repo.head_file(physical_path)?); - let state = import_physical_sqlite_file_state(runtime, physical_path, base.as_ref())?; + let (state, prepared) = prepare_cached_physical_sqlite_file_state( + runtime, + repo, + key, + physical_path, + base.as_ref(), + )?; + tracing::debug!( + key, + page_hash_cache_hit = prepared.page_hash_cache_hit(), + "prepared repository SQLite stage" + ); + file.cache_prepared_sqlite_stage(key.to_string(), prepared); repo.prepare_file_state_path(repo.worktree().join(key), state) .map_err(Into::into) } else if let Some(state) = repo_file_state_for_key(runtime, repo, key)? { diff --git a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs index 1847c353..52deab8d 100644 --- a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs +++ b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs @@ -1,20 +1,438 @@ use graft::volume_writer::VolumeWriter; use rusqlite::{Connection, ErrorCode, OpenFlags, backup::Backup}; -use std::time::{Duration, Instant}; +use std::collections::VecDeque; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; #[cfg(target_os = "macos")] use std::{ ffi::{CString, c_char, c_int}, os::unix::ffi::OsStrExt, }; +use std::{ + fs::OpenOptions, + io::{BufReader, BufWriter}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; use tempfile::TempDir; use super::*; +const PAGE_HASH_CACHE_MAGIC: &[u8; 16] = b"graft-page-index"; +const PAGE_HASH_CACHE_VERSION: u32 = 2; +const PAGE_HASH_BYTES: usize = 32; +const PAGE_HASH_CHUNK_PAGES: usize = 64; +const PAGE_HASH_CHUNK_BYTES: usize = PAGE_HASH_CHUNK_PAGES * PAGESIZE.as_usize(); +const PAGE_HASH_CACHE_HEADER_BYTES: usize = PAGE_HASH_CACHE_MAGIC.len() + 12; +const PAGE_HASH_CACHE_CHECKSUM_BYTES: usize = 32; +const PAGE_SCAN_BUFFER_BYTES: usize = 4 * 1024 * 1024; +const MIN_PAGE_HASH_CACHE_FILE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_PAGE_HASH_CACHE_ENTRIES: usize = 4; +const MAX_WORKTREE_DIFF_PROBES: usize = 16; +const WORKTREE_DIFF_PROBE_VERSION: u32 = 1; +const MAX_PERSISTED_DIFF_PROBE_BYTES: u64 = 64 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct WorktreeFileFingerprint { + len: u64, + modified_nanos: Option, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(unix)] + ctime_seconds: i64, + #[cfg(unix)] + ctime_nanos: i64, +} + +#[derive(Clone, PartialEq, Eq)] +struct WorktreeDiffProbeIdentity { + path: PathBuf, + expected_index: PathBuf, + fingerprint: WorktreeFileFingerprint, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct WorktreeDiffProbe { + matches: bool, + table_candidates: Option>, +} + +#[derive(Serialize, Deserialize)] +struct PersistedWorktreeDiffProbePayload { + version: u32, + fingerprint: WorktreeFileFingerprint, + probe: WorktreeDiffProbe, +} + +#[derive(Serialize, Deserialize)] +struct PersistedWorktreeDiffProbe { + payload: PersistedWorktreeDiffProbePayload, + checksum: String, +} + +static WORKTREE_DIFF_PROBES: OnceLock< + Mutex>, +> = OnceLock::new(); + +fn worktree_diff_probe_identity( + path: &Path, + cache: &SqlitePageHashCache, + expected: &CommitFileState, +) -> Result { + let metadata = std::fs::metadata(path)?; + Ok(WorktreeDiffProbeIdentity { + path: path.to_path_buf(), + expected_index: cache.path_for_state(expected)?, + fingerprint: WorktreeFileFingerprint { + len: metadata.len(), + modified_nanos: metadata.modified().ok().and_then(system_time_nanos), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + #[cfg(unix)] + ctime_seconds: metadata.ctime(), + #[cfg(unix)] + ctime_nanos: metadata.ctime_nsec(), + }, + }) +} + +fn system_time_nanos(time: SystemTime) -> Option { + time.duration_since(UNIX_EPOCH) + .ok() + .map(|value| value.as_nanos()) +} + +fn load_worktree_diff_probe(identity: &WorktreeDiffProbeIdentity) -> Option { + WORKTREE_DIFF_PROBES + .get_or_init(|| Mutex::new(VecDeque::new())) + .lock() + .iter() + .rev() + .find(|(candidate, _)| candidate == identity) + .map(|(_, probe)| probe.clone()) +} + +fn store_worktree_diff_probe(identity: WorktreeDiffProbeIdentity, probe: WorktreeDiffProbe) { + let mut probes = WORKTREE_DIFF_PROBES + .get_or_init(|| Mutex::new(VecDeque::new())) + .lock(); + probes.retain(|(candidate, _)| candidate != &identity); + probes.push_back((identity, probe)); + while probes.len() > MAX_WORKTREE_DIFF_PROBES { + probes.pop_front(); + } +} + +/// A content-addressed page index for one repository `SQLite` path. +/// +/// The file name binds the index to an exact `CommitFileState`. The body is checksummed before any +/// page hashes are trusted, so a missing, stale, truncated, or corrupted cache only disables the +/// optimization and falls back to the authoritative page comparison. +struct SqlitePageHashCache { + directory: PathBuf, +} + +impl SqlitePageHashCache { + fn new(repo: &Repository, key: &str) -> Self { + let key_hash = blake3::hash(key.as_bytes()).to_hex(); + Self { + directory: repo + .graft_dir() + .join("cache") + .join("sqlite-pages") + .join(key_hash.as_str()), + } + } + + fn path_for_state(&self, state: &CommitFileState) -> Result { + let state_hash = sqlite_page_index_state_hash(state)?; + Ok(self.directory.join(format!("pages-v2-{state_hash}.bin"))) + } + + fn probe_path_for_state(&self, state: &CommitFileState) -> Result { + let state_hash = sqlite_page_index_state_hash(state)?; + Ok(self + .directory + .join(format!("worktree-probe-v1-{state_hash}.json"))) + } + + fn load_probe( + &self, + state: &CommitFileState, + fingerprint: &WorktreeFileFingerprint, + ) -> Result, ErrCtx> { + let path = self.probe_path_for_state(state)?; + let metadata = match std::fs::metadata(&path) { + Ok(metadata) if metadata.len() <= MAX_PERSISTED_DIFF_PROBE_BYTES => metadata, + Ok(_) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + debug_assert!(metadata.len() <= MAX_PERSISTED_DIFF_PROBE_BYTES); + let persisted: PersistedWorktreeDiffProbe = serde_json::from_slice(&std::fs::read(path)?) + .map_err(|error| { + ErrCtx::PragmaErr( + format!("failed to decode persisted SQLite worktree probe: {error}").into(), + ) + })?; + let payload_bytes = serde_json::to_vec(&persisted.payload).map_err(|error| { + ErrCtx::PragmaErr( + format!("failed to verify persisted SQLite worktree probe: {error}").into(), + ) + })?; + if persisted.payload.version != WORKTREE_DIFF_PROBE_VERSION + || &persisted.payload.fingerprint != fingerprint + || blake3::hash(&payload_bytes).to_hex().as_str() != persisted.checksum + { + return Ok(None); + } + Ok(Some(persisted.payload.probe)) + } + + fn store_probe( + &self, + state: &CommitFileState, + fingerprint: &WorktreeFileFingerprint, + probe: &WorktreeDiffProbe, + ) -> Result<(), ErrCtx> { + std::fs::create_dir_all(&self.directory)?; + let payload = PersistedWorktreeDiffProbePayload { + version: WORKTREE_DIFF_PROBE_VERSION, + fingerprint: fingerprint.clone(), + probe: probe.clone(), + }; + let payload_bytes = serde_json::to_vec(&payload).map_err(|error| { + ErrCtx::PragmaErr( + format!("failed to encode persisted SQLite worktree probe: {error}").into(), + ) + })?; + let persisted = PersistedWorktreeDiffProbe { + payload, + checksum: blake3::hash(&payload_bytes).to_hex().to_string(), + }; + let bytes = serde_json::to_vec(&persisted).map_err(|error| { + ErrCtx::PragmaErr( + format!("failed to encode persisted SQLite worktree probe: {error}").into(), + ) + })?; + let final_path = self.probe_path_for_state(state)?; + let temp_path = self.directory.join(format!( + ".worktree-probe-v1-{}-{}.tmp", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_nanos()) + )); + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path)?; + let mut writer = BufWriter::new(file); + writer.write_all(&bytes)?; + writer.flush()?; + writer.get_ref().sync_all()?; + drop(writer); + if let Err(error) = std::fs::rename(&temp_path, &final_path) { + let _ = std::fs::remove_file(&temp_path); + if !final_path.exists() { + return Err(error.into()); + } + } + Ok(()) + } + + fn load(&self, state: &CommitFileState) -> Result>, ErrCtx> { + let path = self.path_for_state(state)?; + let expected_page_count = state.snapshot.page_count.to_u32() as usize; + let expected_hash_count = page_hash_chunk_count(expected_page_count); + let expected_len = PAGE_HASH_CACHE_HEADER_BYTES + .checked_add( + expected_hash_count + .checked_mul(PAGE_HASH_BYTES) + .ok_or_else(page_hash_cache_size_error)?, + ) + .and_then(|value| value.checked_add(PAGE_HASH_CACHE_CHECKSUM_BYTES)) + .ok_or_else(page_hash_cache_size_error)?; + let metadata = match std::fs::metadata(&path) { + Ok(metadata) if metadata.len() == expected_len as u64 => metadata, + Ok(_) => return Ok(None), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + debug_assert_eq!(metadata.len(), expected_len as u64); + let bytes = std::fs::read(path)?; + if bytes.len() != expected_len + || &bytes[..PAGE_HASH_CACHE_MAGIC.len()] != PAGE_HASH_CACHE_MAGIC + { + return Ok(None); + } + let version_offset = PAGE_HASH_CACHE_MAGIC.len(); + let version = u32::from_le_bytes( + bytes[version_offset..version_offset + 4] + .try_into() + .expect("page-index version has a fixed width"), + ); + let page_count = u32::from_le_bytes( + bytes[version_offset + 4..version_offset + 8] + .try_into() + .expect("page-index count has a fixed width"), + ); + let chunk_pages = u32::from_le_bytes( + bytes[version_offset + 8..PAGE_HASH_CACHE_HEADER_BYTES] + .try_into() + .expect("page-index chunk size has a fixed width"), + ); + if version != PAGE_HASH_CACHE_VERSION + || page_count as usize != expected_page_count + || chunk_pages as usize != PAGE_HASH_CHUNK_PAGES + { + return Ok(None); + } + let checksum_offset = bytes.len() - PAGE_HASH_CACHE_CHECKSUM_BYTES; + let actual_checksum = blake3::hash(&bytes[..checksum_offset]); + if actual_checksum.as_bytes() != &bytes[checksum_offset..] { + return Ok(None); + } + let mut hashes = Vec::with_capacity(expected_hash_count); + for hash in + bytes[PAGE_HASH_CACHE_HEADER_BYTES..checksum_offset].chunks_exact(PAGE_HASH_BYTES) + { + hashes.push( + hash.try_into() + .expect("validated page-index hashes have a fixed width"), + ); + } + Ok(Some(hashes)) + } + + fn store( + &self, + state: &CommitFileState, + hashes: &[[u8; PAGE_HASH_BYTES]], + ) -> Result<(), ErrCtx> { + let page_count = state.snapshot.page_count.to_u32() as usize; + if hashes.len() != page_hash_chunk_count(page_count) { + return Err(page_hash_cache_size_error()); + } + std::fs::create_dir_all(&self.directory)?; + let final_path = self.path_for_state(state)?; + let temp_path = self.directory.join(format!( + ".pages-v2-{}-{}.tmp", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_nanos()) + )); + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path)?; + let mut writer = BufWriter::new(file); + let mut checksum = blake3::Hasher::new(); + write_page_hash_cache_bytes(&mut writer, &mut checksum, PAGE_HASH_CACHE_MAGIC)?; + write_page_hash_cache_bytes( + &mut writer, + &mut checksum, + &PAGE_HASH_CACHE_VERSION.to_le_bytes(), + )?; + write_page_hash_cache_bytes( + &mut writer, + &mut checksum, + &(page_count as u32).to_le_bytes(), + )?; + write_page_hash_cache_bytes( + &mut writer, + &mut checksum, + &(PAGE_HASH_CHUNK_PAGES as u32).to_le_bytes(), + )?; + for hash in hashes { + write_page_hash_cache_bytes(&mut writer, &mut checksum, hash)?; + } + writer.write_all(checksum.finalize().as_bytes())?; + writer.flush()?; + writer.get_ref().sync_all()?; + drop(writer); + if let Err(error) = std::fs::rename(&temp_path, &final_path) { + let _ = std::fs::remove_file(&temp_path); + if !final_path.exists() { + return Err(error.into()); + } + } + prune_page_hash_cache(&self.directory, &final_path); + Ok(()) + } +} + +fn sqlite_page_index_state_hash(state: &CommitFileState) -> Result { + let encoded = serde_json::to_vec(state).map_err(|error| { + ErrCtx::PragmaErr(format!("failed to encode SQLite page-index state: {error}").into()) + })?; + Ok(blake3::hash(&encoded)) +} + +fn page_hash_cache_size_error() -> ErrCtx { + ErrCtx::PragmaErr("SQLite page-index size exceeds supported limits".into()) +} + +fn write_page_hash_cache_bytes( + writer: &mut BufWriter, + checksum: &mut blake3::Hasher, + bytes: &[u8], +) -> Result<(), ErrCtx> { + writer.write_all(bytes)?; + checksum.update(bytes); + Ok(()) +} + +fn prune_page_hash_cache(directory: &Path, keep: &Path) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + let mut indexes = entries + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name().is_some_and(|name| { + let name = name.to_string_lossy(); + name.starts_with("pages-v") && name.ends_with(".bin") + }) + }) + .collect::>(); + indexes.sort_by_key(|path| { + std::fs::metadata(path) + .and_then(|metadata| metadata.modified()) + .unwrap_or(UNIX_EPOCH) + }); + let remove_count = indexes.len().saturating_sub(MAX_PAGE_HASH_CACHE_ENTRIES); + for path in indexes.into_iter().take(remove_count) { + if path != keep { + if let Some(state_hash) = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix("pages-v2-")) + .and_then(|name| name.strip_suffix(".bin")) + { + let _ = std::fs::remove_file( + directory.join(format!("worktree-probe-v1-{state_hash}.json")), + ); + } + let _ = std::fs::remove_file(path); + } + } +} + +fn page_hash_chunk_count(page_count: usize) -> usize { + page_count.div_ceil(PAGE_HASH_CHUNK_PAGES) +} + /// A stable page reader for a physical `SQLite` worktree file. /// /// This module is the data-plane boundary between repository operations and `SQLite`. Repository /// commands must not read physical `SQLite` pages or manipulate Graft volumes directly. -pub(super) struct PhysicalSqliteReader { +pub(crate) struct PhysicalSqliteReader { input: Mutex, path: PathBuf, snapshot_path: PathBuf, @@ -22,6 +440,86 @@ pub(super) struct PhysicalSqliteReader { _snapshot_dir: Option, } +struct LockedPhysicalSqliteReader { + physical: PhysicalSqliteReader, + _guard: Connection, +} + +impl LockedPhysicalSqliteReader { + /// Reads the live rollback-journal database under one `SQLite` shared lock. + /// + /// This path is only used after an exact page-index hit, so the bounded sequential scan and + /// table mapping finish quickly. Cache misses retain the cloned snapshot path and never hold a + /// worktree lock during their authoritative full comparison. + fn open(path: &Path) -> Result, ErrCtx> { + validate_sqlite_source(path)?; + let source = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + source.busy_timeout(Duration::from_secs(5))?; + let journal_mode: String = + source.pragma_query_value(None, "journal_mode", |row| row.get(0))?; + if !journal_mode.eq_ignore_ascii_case("delete") { + return Ok(None); + } + source.execute_batch("BEGIN")?; + source.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; + let physical = PhysicalSqliteReader::open_snapshot(path, path, None)?; + Ok(Some(Self { physical, _guard: source })) + } +} + +/// Runs a bounded read against a consistent physical `SQLite` image. +/// +/// Rollback-journal databases can be read directly under a shared transaction, which avoids +/// copying an entire large worktree merely to inspect one table. WAL databases retain the online +/// backup fallback because their committed image may span the database and WAL sidecar. +pub(super) fn with_consistent_physical_sqlite_reader( + path: &Path, + read: impl FnOnce(&PhysicalSqliteReader) -> Result, +) -> Result { + if let Some(locked) = LockedPhysicalSqliteReader::open(path)? { + return read(&locked.physical); + } + let snapshot = PhysicalSqliteReader::open(path)?; + read(&snapshot) +} + +/// The stable `SQLite` image and page-set produced by one staging operation. +/// +/// Keeping this alive until commit mirrors Git's index semantics: the staged snapshot, not the +/// possibly newer worktree, is the source of truth for the commit. The captured changed-page set +/// also avoids repeating the full physical/staged/previous comparison during summary generation. +pub(crate) struct PreparedSqliteStage { + physical: Option, + prepared_table_candidates: Option>>, + previous: Option, + staged: CommitFileState, + changed_pages: BTreeSet, + page_hash_cache_hit: bool, +} + +impl PreparedSqliteStage { + pub(crate) fn matches(&self, previous: &CommitFileState, staged: &CommitFileState) -> bool { + self.previous.as_ref() == Some(previous) && &self.staged == staged + } + + pub(crate) fn table_candidates(&self) -> Result>, ErrCtx> { + if let Some(candidates) = &self.prepared_table_candidates { + return Ok(candidates.clone()); + } + self.physical + .as_ref() + .expect("deferred table candidates retain their stable SQLite snapshot") + .table_candidates_for_changed_pages(&self.changed_pages) + } + + pub(crate) fn page_hash_cache_hit(&self) -> bool { + self.page_hash_cache_hit + } +} + impl PhysicalSqliteReader { pub(super) fn open(path: &Path) -> Result { validate_sqlite_source(path)?; @@ -87,6 +585,28 @@ impl PhysicalSqliteReader { RepoWorktreeFileState { page_count: self.page_count() } } + fn visit_page_chunks( + &self, + mut visitor: impl FnMut(u32, &[u8]) -> Result<(), ErrCtx>, + ) -> Result<(), ErrCtx> { + let mut input = self.input.lock(); + input.seek(SeekFrom::Start(0))?; + let mut reader = BufReader::with_capacity(PAGE_SCAN_BUFFER_BYTES, &mut *input); + let mut chunk = vec![0_u8; PAGE_HASH_CHUNK_BYTES]; + let mut first_page = 1_u32; + let page_count = self.page_count().to_u32(); + while first_page <= page_count { + graft::repo::cancellation_checkpoint()?; + let pages_remaining = (page_count - first_page + 1) as usize; + let chunk_pages = pages_remaining.min(PAGE_HASH_CHUNK_PAGES); + let chunk_bytes = chunk_pages * PAGESIZE.as_usize(); + reader.read_exact(&mut chunk[..chunk_bytes])?; + visitor(first_page, &chunk[..chunk_bytes])?; + first_page += chunk_pages as u32; + } + Ok(()) + } + /// Finds tables that own pages changed from `expected` to this stable worktree snapshot. /// /// `dbstat` gives us `SQLite`'s page ownership without decoding every row. Returning `None` @@ -117,6 +637,27 @@ impl PhysicalSqliteReader { self.table_candidates_for_changed_pages(&changed_pages) } + /// Finds changed table candidates using the committed page index when it is available. + /// + /// This mirrors Git's index/stat fast path: hash the worktree sequentially in coarse chunks, + /// then compare individual pages only inside chunks whose content hash changed. Falling back + /// to the authoritative page-by-page comparison preserves correctness when the index is + /// missing or unreadable. + pub(super) fn cached_changed_table_candidates( + &self, + runtime: &Runtime, + repo: &Repository, + key: &str, + expected_state: &CommitFileState, + expected: &dyn VolumeRead, + ) -> Result>, ErrCtx> { + let probe = self.cached_diff_probe(runtime, repo, key, expected_state, expected)?; + if !probe.matches && probe.table_candidates.is_none() { + return self.changed_table_candidates(expected); + } + Ok(probe.table_candidates) + } + /// Reuses the physical worktree as a fast staged-snapshot reader only when every byte still /// matches the staged state. While validating that invariant, collect pages changed from the /// previous commit so checkpoint summaries can avoid scanning unrelated large tables. @@ -224,6 +765,115 @@ impl PhysicalSqliteReader { } Ok(true) } + + /// Verifies the worktree against an exact committed state using its persisted page index. + /// + /// A cache hit turns the comparison into one sequential physical read and avoids both a + /// worktree clone and random reads from the immutable Graft snapshot. The page hashes are + /// content-addressed by `expected`; any cache miss or validation failure uses the authoritative + /// page comparison. + pub(super) fn matches_cached_state( + &self, + runtime: &Runtime, + repo: &Repository, + key: &str, + expected: &CommitFileState, + ) -> Result { + let expected_reader = runtime.snapshot_reader(expected.snapshot.to_snapshot()); + self.cached_diff_probe(runtime, repo, key, expected, &expected_reader) + .map(|probe| probe.matches) + } + + fn cached_diff_probe( + &self, + runtime: &Runtime, + repo: &Repository, + key: &str, + expected_state: &CommitFileState, + expected: &dyn VolumeRead, + ) -> Result { + if self.snapshot_path != self.path { + return Ok(WorktreeDiffProbe { + matches: self.matches_state(runtime, expected_state)?, + table_candidates: None, + }); + } + let cache = SqlitePageHashCache::new(repo, key); + let identity = worktree_diff_probe_identity(&self.path, &cache, expected_state)?; + if let Some(probe) = load_worktree_diff_probe(&identity) { + return Ok(probe); + } + match cache.load_probe(expected_state, &identity.fingerprint) { + Ok(Some(probe)) => { + store_worktree_diff_probe(identity, probe.clone()); + return Ok(probe); + } + Ok(None) => {} + Err(error) => { + tracing::debug!( + ?error, + "ignoring unreadable persisted SQLite worktree probe" + ); + } + } + let cached_hashes = match cache.load(expected_state) { + Ok(Some(hashes)) => hashes, + Ok(None) => { + return Ok(WorktreeDiffProbe { + matches: self.matches_state(runtime, expected_state)?, + table_candidates: None, + }); + } + Err(error) => { + tracing::debug!(?error, "ignoring unreadable SQLite page-index cache"); + return Ok(WorktreeDiffProbe { + matches: self.matches_state(runtime, expected_state)?, + table_candidates: None, + }); + } + }; + + let mut changed_pages = BTreeSet::new(); + self.visit_page_chunks(|first_page, chunk_bytes| { + let chunk_index = (first_page - 1) as usize / PAGE_HASH_CHUNK_PAGES; + let current_hash = blake3::hash(chunk_bytes); + if cached_hashes + .get(chunk_index) + .is_some_and(|expected_hash| expected_hash == current_hash.as_bytes()) + { + return Ok(()); + } + + for (page_offset, page_bytes) in + chunk_bytes.chunks_exact(PAGESIZE.as_usize()).enumerate() + { + let page_number = first_page + page_offset as u32; + let pageidx = PageIdx::try_from(page_number).map_err(|error| { + ErrCtx::PragmaErr( + format!("invalid SQLite page index {page_number}: {error}").into(), + ) + })?; + let unchanged = expected.page_count().contains(pageidx) + && expected.read_page(pageidx)?.as_ref() == page_bytes; + if !unchanged { + changed_pages.insert(page_number); + } + } + Ok(()) + })?; + for page_number in (self.page_count().to_u32() + 1)..=expected.page_count().to_u32() { + changed_pages.insert(page_number); + } + let probe = WorktreeDiffProbe { + matches: changed_pages.is_empty(), + table_candidates: self.table_candidates_for_changed_pages(&changed_pages)?, + }; + if let Err(error) = cache.store_probe(expected_state, &identity.fingerprint, &probe) { + tracing::debug!(?error, "failed to persist SQLite worktree probe"); + } + store_worktree_diff_probe(identity, probe.clone()); + Ok(probe) + } } fn validate_sqlite_source(path: &Path) -> Result<(), ErrCtx> { @@ -596,7 +1246,88 @@ pub(super) fn import_physical_sqlite_file_state( base: Option<&CommitFileState>, ) -> Result { let physical = PhysicalSqliteReader::open(path)?; - import_sqlite_reader_state(runtime, path, base, physical) + import_sqlite_reader_state(runtime, path, base, &physical, None, None) + .map(|(state, _, _)| state) +} + +#[cfg(test)] +pub(super) fn prepare_physical_sqlite_file_state( + runtime: &Runtime, + path: &Path, + base: Option<&CommitFileState>, +) -> Result<(CommitFileState, PreparedSqliteStage), ErrCtx> { + prepare_physical_sqlite_file_state_with_cache(runtime, path, base, None) +} + +pub(super) fn prepare_cached_physical_sqlite_file_state( + runtime: &Runtime, + repo: &Repository, + key: &str, + path: &Path, + base: Option<&CommitFileState>, +) -> Result<(CommitFileState, PreparedSqliteStage), ErrCtx> { + if std::fs::metadata(path)?.len() < MIN_PAGE_HASH_CACHE_FILE_BYTES { + return prepare_physical_sqlite_file_state_with_cache(runtime, path, base, None); + } + let cache = SqlitePageHashCache::new(repo, key); + prepare_physical_sqlite_file_state_with_cache(runtime, path, base, Some(&cache)) +} + +fn prepare_physical_sqlite_file_state_with_cache( + runtime: &Runtime, + path: &Path, + base: Option<&CommitFileState>, + cache: Option<&SqlitePageHashCache>, +) -> Result<(CommitFileState, PreparedSqliteStage), ErrCtx> { + let cached_hashes = match (base, cache) { + (Some(base), Some(cache)) => match cache.load(base) { + Ok(hashes) => hashes, + Err(error) => { + tracing::debug!(?error, "ignoring unreadable SQLite page-index cache"); + None + } + }, + _ => None, + }; + if cached_hashes.is_some() + && let Some(locked) = LockedPhysicalSqliteReader::open(path)? + { + let (state, changed_pages, page_hash_cache_hit) = import_sqlite_reader_state( + runtime, + path, + base, + &locked.physical, + cache, + cached_hashes, + )?; + let prepared_table_candidates = Some( + locked + .physical + .table_candidates_for_changed_pages(&changed_pages)?, + ); + let prepared = PreparedSqliteStage { + physical: None, + prepared_table_candidates, + previous: base.cloned(), + staged: state.clone(), + changed_pages, + page_hash_cache_hit, + }; + return Ok((state, prepared)); + } + + let physical = PhysicalSqliteReader::open(path)?; + let (state, changed_pages, page_hash_cache_hit) = + import_sqlite_reader_state(runtime, path, base, &physical, cache, cached_hashes)?; + let prepared = PreparedSqliteStage { + physical: Some(physical), + prepared_table_candidates: None, + previous: base.cloned(), + staged: state.clone(), + changed_pages, + page_hash_cache_hit, + }; + Ok((state, prepared)) } pub(super) fn import_stable_sqlite_file_state( @@ -604,51 +1335,97 @@ pub(super) fn import_stable_sqlite_file_state( path: &Path, ) -> Result { let physical = PhysicalSqliteReader::open_stable(path)?; - import_sqlite_reader_state(runtime, path, None, physical) + import_sqlite_reader_state(runtime, path, None, &physical, None, None) + .map(|(state, _, _)| state) } fn import_sqlite_reader_state( runtime: &Runtime, path: &Path, base: Option<&CommitFileState>, - physical: PhysicalSqliteReader, -) -> Result { + physical: &PhysicalSqliteReader, + cache: Option<&SqlitePageHashCache>, + cached_hashes: Option>, +) -> Result<(CommitFileState, BTreeSet, bool), ErrCtx> { + let page_hash_cache_hit = cached_hashes.is_some(); let base_reader = base.map(|state| runtime.snapshot_reader(state.snapshot.to_snapshot())); let mut target = None; + let mut changed_pages = BTreeSet::new(); + let page_count = physical.page_count().to_u32() as usize; + let mut current_hashes = Vec::with_capacity(page_hash_chunk_count(page_count)); + + physical.visit_page_chunks(|first_page, chunk_bytes| { + let current_hash = *blake3::hash(chunk_bytes).as_bytes(); + current_hashes.push(current_hash); + let chunk_index = (first_page - 1) as usize / PAGE_HASH_CHUNK_PAGES; + if cached_hashes + .as_ref() + .and_then(|hashes| hashes.get(chunk_index)) + .is_some_and(|expected| expected == ¤t_hash) + { + return Ok(()); + } - for page_number in 1..=physical.page_count().to_u32() { - graft::repo::cancellation_checkpoint()?; - let pageidx = PageIdx::try_from(page_number).map_err(|err| { - ErrCtx::PragmaErr( - format!("invalid SQLite page index in `{}`: {err}", path.display()).into(), - ) - })?; - let page = physical.read_page(pageidx)?; - let unchanged = match &base_reader { - Some(reader) if reader.page_count().contains(pageidx) => { - reader.read_page(pageidx)? == page + for (page_offset, page_bytes) in chunk_bytes.chunks_exact(PAGESIZE.as_usize()).enumerate() { + let page_number = first_page + page_offset as u32; + let pageidx = PageIdx::try_from(page_number).map_err(|error| { + ErrCtx::PragmaErr( + format!("invalid SQLite page index in `{}`: {error}", path.display()).into(), + ) + })?; + let unchanged = match &base_reader { + Some(reader) if reader.page_count().contains(pageidx) => { + reader.read_page(pageidx)?.as_ref() == page_bytes + } + _ => false, + }; + if unchanged { + continue; } - _ => false, - }; - if unchanged { - continue; + changed_pages.insert(page_number); + let target = ensure_import_target(runtime, base, &mut target)?; + let page = Page::try_from(page_bytes).map_err(|error| { + ErrCtx::PragmaErr( + format!("invalid SQLite page in `{}`: {error}", path.display()).into(), + ) + })?; + target.writer.write_page(pageidx, page)?; } - - let target = ensure_import_target(runtime, base, &mut target)?; - target.writer.write_page(pageidx, page)?; - } + Ok(()) + })?; if base.is_none_or(|state| state.snapshot.page_count != physical.page_count()) { + if let Some(base) = base { + for page_number in + (physical.page_count().to_u32() + 1)..=base.snapshot.page_count.to_u32() + { + changed_pages.insert(page_number); + } + } let target = ensure_import_target(runtime, base, &mut target)?; target.writer.soft_truncate(physical.page_count())?; } - let Some(target) = target else { - return Ok(base + let state = match target { + Some(target) => target.commit(runtime)?, + None => base .cloned() - .expect("an unchanged import must have a base snapshot")); + .expect("an unchanged import must have a base snapshot"), }; - target.commit(runtime) + if let Some(cache) = cache + && let Err(error) = cache.store(&state, ¤t_hashes) + { + tracing::debug!(?error, "failed to persist SQLite page-index cache"); + } + tracing::debug!( + path = %path.display(), + page_count, + chunks_hashed = current_hashes.len(), + changed_pages = changed_pages.len(), + page_hash_cache_hit, + "prepared physical SQLite state" + ); + Ok((state, changed_pages, page_hash_cache_hit)) } struct ImportTarget { @@ -766,6 +1543,17 @@ mod tests { connection } + fn prepare_with_forced_page_cache( + runtime: &Runtime, + repo: &Repository, + key: &str, + path: &Path, + base: Option<&CommitFileState>, + ) -> Result<(CommitFileState, PreparedSqliteStage), ErrCtx> { + let cache = SqlitePageHashCache::new(repo, key); + prepare_physical_sqlite_file_state_with_cache(runtime, path, base, Some(&cache)) + } + #[test] fn unchanged_import_reuses_snapshot_and_changed_import_is_incremental() { let temp = tempfile::tempdir().unwrap(); @@ -837,6 +1625,260 @@ mod tests { ); } + #[test] + fn prepared_stage_keeps_candidates_after_worktree_moves_on() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (staged, prepared) = + prepare_physical_sqlite_file_state(&runtime, &path, Some(&initial)).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + + assert!(prepared.matches(&initial, &staged)); + assert_eq!( + prepared.table_candidates().unwrap(), + Some(BTreeSet::from(["records".to_string()])) + ); + } + + #[test] + fn cached_stage_reuses_persisted_page_hashes_for_the_exact_base() { + let temp = tempfile::tempdir().unwrap(); + let repo = Repository::init(temp.path()).unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (first, first_prepared) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&initial)) + .unwrap(); + assert!(!first_prepared.page_hash_cache_hit()); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + let (second, second_prepared) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&first)) + .unwrap(); + assert!(second_prepared.page_hash_cache_hit()); + assert!(second_prepared.physical.is_none()); + assert!(second_prepared.prepared_table_candidates.is_some()); + assert_ne!(second.snapshot, first.snapshot); + + let unchanged = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&second)) + .unwrap(); + assert!(unchanged.1.page_hash_cache_hit()); + assert_eq!(unchanged.0, second); + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 1", + [vec![0xC3_u8; 3_000]], + ) + .expect("the indexed read lock must be released before stage returns"); + } + + #[test] + fn small_database_skips_the_persistent_page_hash_cache() { + let temp = tempfile::tempdir().unwrap(); + let repo = Repository::init(temp.path()).unwrap(); + let path = temp.path().join("small.sqlite"); + let connection = create_database(&path, "delete"); + assert!(std::fs::metadata(&path).unwrap().len() < MIN_PAGE_HASH_CACHE_FILE_BYTES); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (_, prepared) = prepare_cached_physical_sqlite_file_state( + &runtime, + &repo, + "small.sqlite", + &path, + Some(&initial), + ) + .unwrap(); + + assert!(!prepared.page_hash_cache_hit()); + assert!(!repo.graft_dir().join("cache").join("sqlite-pages").exists()); + } + + #[test] + fn cached_changed_table_candidates_reuses_the_committed_page_index() { + let temp = tempfile::tempdir().unwrap(); + let repo = Repository::init(temp.path()).unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (committed, _) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&initial)) + .unwrap(); + + let matches_committed = with_consistent_physical_sqlite_reader(&path, |physical| { + physical.matches_cached_state(&runtime, &repo, "app.sqlite", &committed) + }) + .unwrap(); + assert!(matches_committed); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + let expected = runtime.snapshot_reader(committed.snapshot.to_snapshot()); + let candidates = with_consistent_physical_sqlite_reader(&path, |physical| { + physical.cached_changed_table_candidates( + &runtime, + &repo, + "app.sqlite", + &committed, + &expected, + ) + }) + .unwrap(); + + assert_eq!(candidates, Some(BTreeSet::from(["records".to_string()]))); + let cache = SqlitePageHashCache::new(&repo, "app.sqlite"); + assert!(cache.probe_path_for_state(&committed).unwrap().is_file()); + WORKTREE_DIFF_PROBES + .get_or_init(|| Mutex::new(VecDeque::new())) + .lock() + .clear(); + let matches_changed = with_consistent_physical_sqlite_reader(&path, |physical| { + physical.matches_cached_state(&runtime, &repo, "app.sqlite", &committed) + }) + .unwrap(); + assert!(!matches_changed); + } + + #[test] + fn bounded_consistent_reader_uses_the_live_rollback_journal_image() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + + let read_live_image = + with_consistent_physical_sqlite_reader( + &path, + |reader| Ok(reader.snapshot_path == path), + ) + .unwrap(); + + assert!(read_live_image); + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 1", + [vec![0xC3_u8; 3_000]], + ) + .expect("the bounded read lock must be released after the callback"); + } + + #[test] + fn corrupted_page_hash_cache_falls_back_to_authoritative_comparison() { + let temp = tempfile::tempdir().unwrap(); + let repo = Repository::init(temp.path()).unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "delete"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (first, _) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&initial)) + .unwrap(); + let cache = SqlitePageHashCache::new(&repo, "app.sqlite"); + std::fs::write(cache.path_for_state(&first).unwrap(), b"corrupted").unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + let (second, prepared) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&first)) + .unwrap(); + assert!(!prepared.page_hash_cache_hit()); + assert_ne!(second.snapshot, first.snapshot); + } + + #[test] + fn cached_wal_stage_retains_the_online_backup_fallback() { + let temp = tempfile::tempdir().unwrap(); + let repo = Repository::init(temp.path()).unwrap(); + let path = temp.path().join("app.sqlite"); + let connection = create_database(&path, "wal"); + let runtime = test_runtime(); + let initial = import_physical_sqlite_file_state(&runtime, &path, None).unwrap(); + + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 32", + [vec![0xA5_u8; 3_000]], + ) + .unwrap(); + let (first, _) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&initial)) + .unwrap(); + connection + .execute( + "UPDATE records SET payload = ?1 WHERE id = 48", + [vec![0x5A_u8; 3_000]], + ) + .unwrap(); + let (second, prepared) = + prepare_with_forced_page_cache(&runtime, &repo, "app.sqlite", &path, Some(&first)) + .unwrap(); + + assert!(prepared.page_hash_cache_hit()); + assert!(prepared.physical.is_some()); + assert!(prepared.prepared_table_candidates.is_none()); + assert_ne!(second.snapshot, first.snapshot); + } + #[test] fn wal_import_reads_committed_state_without_checkpointing_source() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/graft-sqlite/src/row_level_diff.rs b/crates/graft-sqlite/src/row_level_diff.rs index 3b29a4ba..c311b880 100644 --- a/crates/graft-sqlite/src/row_level_diff.rs +++ b/crates/graft-sqlite/src/row_level_diff.rs @@ -800,6 +800,72 @@ pub fn bounded_row_level_diff_readers_for_summary_tables( ) } +/// Uses the existing page-aware rowid diff when every candidate table has an ordinary rowid +/// layout. For these tables it is faster to decode only rows on changed B-tree pages than to merge +/// the complete rowid stream. `WITHOUT ROWID` and schema-changing tables return `None` so callers +/// retain the bounded primary-key path that avoids materializing large Eidos tables. +pub fn rowid_table_summaries_for_tables( + from_reader: &dyn VolumeRead, + to_reader: &dyn VolumeRead, + from_lsn: LSN, + to_lsn: LSN, + tables: &BTreeSet, +) -> Result>, graft::err::GraftErr> { + if sqlite_page_size(from_reader)? != PAGESIZE.as_u32() + || sqlite_page_size(to_reader)? != PAGESIZE.as_u32() + { + return Ok(None); + } + let from_scanner = TableScanner::new(from_reader).map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to parse source B-tree: {error:?}")) + })?; + let to_scanner = TableScanner::new(to_reader).map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to parse target B-tree: {error:?}")) + })?; + let from_master = from_scanner.read_master_table().map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to read source schema: {error:?}")) + })?; + let to_master = to_scanner.read_master_table().map_err(|error| { + graft::err::LogicalErr::Other(format!("Failed to read target schema: {error:?}")) + })?; + for table in tables { + let from_entry = from_master.iter().find(|entry| entry.name == *table); + let to_entry = to_master.iter().find(|entry| entry.name == *table); + let stable_rowid_layout = match (from_entry, to_entry) { + (Some(from), Some(to)) => { + !is_without_rowid_table(from) && !is_without_rowid_table(to) && from.sql == to.sql + } + _ => false, + }; + if !stable_rowid_layout { + return Ok(None); + } + } + + let mut summaries = Vec::new(); + for table in tables { + let diff = row_level_diff_readers_for_table( + from_reader, + to_reader, + from_lsn, + to_lsn, + Some(table), + )?; + for changes in diff.table_changes { + let (inserts, deletes, updates) = count_changes(&changes.changes); + if inserts + deletes + updates > 0 { + summaries.push(BoundedTableSummary { + table_name: changes.table_name, + inserts, + deletes, + updates, + }); + } + } + } + Ok(Some(summaries)) +} + fn bounded_row_diff_from_readers( from_reader: &dyn VolumeRead, to_reader: &dyn VolumeRead, diff --git a/crates/graft/src/core/commit_hash.rs b/crates/graft/src/core/commit_hash.rs index ca568073..6f1ad615 100644 --- a/crates/graft/src/core/commit_hash.rs +++ b/crates/graft/src/core/commit_hash.rs @@ -53,6 +53,7 @@ pub enum CommitHashParseErr { Copy, PartialEq, Eq, + Hash, Default, TryFromBytes, IntoBytes, @@ -67,7 +68,7 @@ pub enum CommitHashPrefix { } #[derive( - Clone, PartialEq, Eq, Default, TryFromBytes, IntoBytes, Immutable, KnownLayout, Unaligned, + Clone, PartialEq, Eq, Hash, Default, TryFromBytes, IntoBytes, Immutable, KnownLayout, Unaligned, )] #[repr(C)] pub struct CommitHash { diff --git a/crates/graft/src/local/fjall_storage.rs b/crates/graft/src/local/fjall_storage.rs index 2c88e299..185b56a1 100644 --- a/crates/graft/src/local/fjall_storage.rs +++ b/crates/graft/src/local/fjall_storage.rs @@ -2,7 +2,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, ops::RangeInclusive, - path::Path, + path::{Path, PathBuf}, }; use crate::{ @@ -114,9 +114,25 @@ impl Keyspaces { } } +fn hydrated_snapshot_key(snapshot: &Snapshot) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"graft-hydrated-snapshot-v1\0"); + hasher.update(&snapshot.page_count.to_u32().to_le_bytes()); + for range in snapshot.iter() { + hasher.update(range.log.serialize().as_bytes()); + hasher.update(&range.lsns.start().to_u64().to_le_bytes()); + hasher.update(&range.lsns.end().to_u64().to_le_bytes()); + } + hasher.finalize().to_hex().to_string() +} + pub struct FjallStorage { db: fjall::Database, ks: Keyspaces, + /// Derived snapshot-presence proofs live outside Fjall's keyspaces so checking or recording + /// one never enlarges the database's write set. This is deliberately analogous to Git's + /// replaceable commit-graph/index helpers: deleting the directory only loses acceleration. + hydration_cache_dir: PathBuf, /// Must be held while performing read+write transactions. /// Read-only and write-only transactions don't need to hold the lock as @@ -134,26 +150,33 @@ impl Debug for FjallStorage { impl FjallStorage { pub fn open>(path: P) -> Result { - let builder = Database::builder(path); + let path = path.as_ref().to_path_buf(); + let builder = Database::builder(&path); #[cfg(target_arch = "wasm32")] let builder = builder.worker_threads_unchecked(0); - Self::open_from_builder(builder) + Self::open_from_builder(builder, path.join("cache/hydrated-snapshots-v1")) } pub fn open_temporary() -> Result { let path = tempfile::tempdir()?.keep(); - let builder = Database::builder(path).temporary(true); + let builder = Database::builder(&path).temporary(true); #[cfg(target_arch = "wasm32")] let builder = builder.worker_threads_unchecked(0); - Self::open_from_builder(builder) + Self::open_from_builder(builder, path.join("cache/hydrated-snapshots-v1")) } fn open_from_builder( builder: fjall::DatabaseBuilder, + hydration_cache_dir: PathBuf, ) -> Result { let db = builder.open()?; let ks = Keyspaces::open(&db)?; - Ok(Self { db, ks, lock: Default::default() }) + Ok(Self { + db, + ks, + hydration_cache_dir, + lock: Default::default(), + }) } pub(crate) fn read(&self) -> ReadGuard<'_> { @@ -171,6 +194,46 @@ impl FjallStorage { ReadWriteGuard::open(self) } + pub(crate) fn snapshot_hydration_cached( + &self, + snapshot: &Snapshot, + ) -> Result { + Ok(self + .hydration_cache_dir + .join(hydrated_snapshot_key(snapshot)) + .is_file()) + } + + pub(crate) fn mark_snapshot_hydrated( + &self, + snapshot: &Snapshot, + ) -> Result<(), FjallStorageErr> { + std::fs::create_dir_all(&self.hydration_cache_dir)?; + let final_path = self + .hydration_cache_dir + .join(hydrated_snapshot_key(snapshot)); + if final_path.is_file() { + return Ok(()); + } + let temp_path = self.hydration_cache_dir.join(format!( + ".hydrated-{}-{}.tmp", + std::process::id(), + rand::random::() + )); + std::fs::write(&temp_path, b"hydrated\n")?; + match std::fs::rename(&temp_path, &final_path) { + Ok(()) => Ok(()), + Err(_error) if final_path.is_file() => { + let _ = std::fs::remove_file(temp_path); + Ok(()) + } + Err(error) => { + let _ = std::fs::remove_file(temp_path); + Err(error.into()) + } + } + } + pub fn write_page( &self, sid: SegmentId, @@ -382,6 +445,14 @@ impl FjallStorage { return Ok(outcome); } + // Hydration markers are derived indexes. Clear them before deleting + // pages so a later Runtime never trusts a marker invalidated by GC. + match std::fs::remove_dir_all(&self.hydration_cache_dir) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let mut batch = self.db.batch(); for vid in candidate_volumes { batch.remove_typed(&self.ks.volumes, vid); @@ -408,7 +479,6 @@ impl FjallStorage { self.ks.checkpoints.major_compact()?; self.ks.page_versions.major_compact()?; self.ks.pages.major_compact()?; - Ok(outcome) } } @@ -900,6 +970,17 @@ impl<'a> ReadWriteGuard<'a> { return Err(LogicalErr::VolumeConcurrentWrite(vid.clone()).into()); } + // Record local completeness in the replaceable side cache after the commit becomes + // durable. Unlike the former Fjall keyspace this does not enlarge every storage write + // transaction, while the next diff/commit can still avoid proving pages Graft just wrote. + let base_is_fully_hydrated = self + .read + .storage + .snapshot_hydration_cached(&snapshot) + .unwrap_or(false); + let commit_is_fully_hydrated = + page_count.to_usize() == pages.len() || base_is_fully_hydrated; + let volume = self.read.volume(vid)?; // the commit_lsn is the next lsn for the volume's local Log @@ -944,7 +1025,13 @@ impl<'a> ReadWriteGuard<'a> { // since we are holding a read_write lock, we know that no other thread // is concurrently committing to the volume, so we know this snapshot // will reflect the commit we just executed - self.read.storage.read().snapshot(&volume.vid) + let snapshot = self.read.storage.read().snapshot(&volume.vid)?; + if commit_is_fully_hydrated + && let Err(error) = self.read.storage.mark_snapshot_hydrated(&snapshot) + { + tracing::debug!(?error, "failed to record local snapshot hydration proof"); + } + Ok(snapshot) } /// Verify we are ready to make a remote commit and update the volume diff --git a/crates/graft/src/repo.rs b/crates/graft/src/repo.rs index ebfce228..f0afc18f 100644 --- a/crates/graft/src/repo.rs +++ b/crates/graft/src/repo.rs @@ -1016,13 +1016,13 @@ pub struct SwitchNewBranchPlan { pub checkout: CheckoutPlan, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RepoSnapshot { pub page_count: PageCount, pub ranges: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RepoLogRange { pub log: LogId, pub start: LSN, @@ -1031,7 +1031,7 @@ pub struct RepoLogRange { pub commits: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RepoStorageCommit { pub lsn: LSN, pub commit_hash: CommitHash, diff --git a/crates/graft/src/repo/inventory.rs b/crates/graft/src/repo/inventory.rs index 247ad03c..ef12b80d 100644 --- a/crates/graft/src/repo/inventory.rs +++ b/crates/graft/src/repo/inventory.rs @@ -1,6 +1,35 @@ use super::*; impl Repository { + /// Refreshes repository/ref metadata on an existing status without reclassifying the + /// worktree. Embedders can reuse an already-proven local worktree status while still + /// observing fetch/push updates to remote-tracking refs. + pub fn refresh_status_repository_projection(&self, status: &mut RepoStatus) -> Result<()> { + let config = self.config()?; + let head = self.head()?; + let upstream = head + .branch_name() + .map(|branch| self.branch_upstream(branch)) + .transpose()? + .flatten(); + let head_target = self.head_target()?; + let upstream_status = self.upstream_status(head_target.as_deref(), upstream.as_ref())?; + + status.repository_format_version = config.core.repository_format_version; + status.head = head; + status.head_target = head_target; + status.merge_head = self.merge_head()?; + status.orig_head = self.orig_head()?; + status.branches = self.branches()?; + status.remotes = self.remotes()?; + status.upstream = upstream; + status.ahead = upstream_status.as_ref().map_or(0, |value| value.ahead); + status.behind = upstream_status.as_ref().map_or(0, |value| value.behind); + status.upstream_status = upstream_status; + status.refresh_summary_flags(); + Ok(()) + } + pub fn status(&self) -> Result { let config = self.config()?; let head = self.head()?; diff --git a/crates/graft/src/rt/action/hydrate_snapshot.rs b/crates/graft/src/rt/action/hydrate_snapshot.rs index 55939a9d..23b8cb0a 100644 --- a/crates/graft/src/rt/action/hydrate_snapshot.rs +++ b/crates/graft/src/rt/action/hydrate_snapshot.rs @@ -21,6 +21,9 @@ pub struct HydrateSnapshot { impl Action for HydrateSnapshot { async fn run(self, storage: Arc, remote: Arc) -> Result<(), GraftErr> { + if storage.snapshot_hydration_cached(&self.snapshot)? { + return Ok(()); + } let missing_frames = storage.read().find_missing_frames(&self.snapshot)?; futures::stream::iter( missing_frames @@ -32,6 +35,8 @@ impl Action for HydrateSnapshot { .try_for_each_concurrent(HYDRATE_CONCURRENCY, |range| { FetchSegment { range }.run(storage.clone(), remote.clone()) }) - .await + .await?; + storage.mark_snapshot_hydrated(&self.snapshot)?; + Ok(()) } } diff --git a/crates/graft/src/rt/runtime.rs b/crates/graft/src/rt/runtime.rs index b491d12d..06c2bb55 100644 --- a/crates/graft/src/rt/runtime.rs +++ b/crates/graft/src/rt/runtime.rs @@ -1,4 +1,13 @@ -use std::{collections::BTreeSet, ops::RangeInclusive, path::Path, sync::Arc, time::Duration}; +use std::{ + collections::BTreeSet, + ops::RangeInclusive, + path::Path, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; use crate::core::{ CommitHashBuilder, LogId, PageCount, PageIdx, SegmentId, VolumeId, @@ -63,8 +72,11 @@ pub struct Runtime { inner: Arc, } +static NEXT_RUNTIME_INSTANCE_ID: AtomicU64 = AtomicU64::new(1); + #[derive(Debug)] struct RuntimeInner { + instance_id: u64, tokio: tokio::runtime::Handle, storage: Arc, remote: Arc, @@ -90,10 +102,21 @@ impl Runtime { )); } Runtime { - inner: Arc::new(RuntimeInner { tokio: tokio_rt, storage, remote }), + inner: Arc::new(RuntimeInner { + instance_id: NEXT_RUNTIME_INSTANCE_ID.fetch_add(1, Ordering::Relaxed), + tokio: tokio_rt, + storage, + remote, + }), } } + /// Returns an identifier shared by clones of this runtime and unique to its + /// backing runtime instance for the lifetime of this process. + pub fn instance_id(&self) -> u64 { + self.inner.instance_id + } + pub(crate) fn storage(&self) -> &FjallStorage { &self.inner.storage } @@ -381,6 +404,12 @@ impl Runtime { self.run_action(HydrateSnapshot { snapshot }) } + /// Returns whether a previous complete hydration of this exact snapshot is + /// recorded in the local storage index. + pub fn snapshot_hydration_cached(&self, snapshot: &Snapshot) -> Result { + Ok(self.storage().snapshot_hydration_cached(snapshot)?) + } + pub fn snapshot_hydrate_from(&self, snapshot: Snapshot, remote: Arc) -> Result<()> { self.snapshot_fetch_from(&snapshot, remote.clone())?; self.run_action_with_remote(HydrateSnapshot { snapshot }, remote) @@ -580,7 +609,10 @@ impl Runtime { mod tests { use std::{collections::BTreeSet, sync::Arc, time::Duration}; - use crate::core::{LogId, PageIdx, lsn::LSN, page::Page}; + use crate::{ + core::{LogId, PageIdx, lsn::LSN, page::Page}, + snapshot::Snapshot, + }; use test_log::test; use tokio::time::sleep; @@ -589,6 +621,66 @@ mod tests { volume_reader::VolumeRead, volume_writer::VolumeWrite, }; + #[test] + fn snapshot_hydration_marker_persists_and_gc_invalidates_it() { + let tokio_rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let remote = Arc::new(RemoteConfig::Memory.build().unwrap()); + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(FjallStorage::open(directory.path()).unwrap()); + let runtime = Runtime::new( + tokio_rt.handle().clone(), + remote.clone(), + storage.clone(), + None, + ); + let snapshot = Snapshot::empty(); + + assert!(!runtime.snapshot_hydration_cached(&snapshot).unwrap()); + runtime.snapshot_hydrate(snapshot.clone()).unwrap(); + assert!(runtime.snapshot_hydration_cached(&snapshot).unwrap()); + + drop(runtime); + drop(storage); + let reopened = Arc::new(FjallStorage::open(directory.path()).unwrap()); + let runtime = Runtime::new(tokio_rt.handle().clone(), remote, reopened, None); + assert!(runtime.snapshot_hydration_cached(&snapshot).unwrap()); + + runtime.storage_gc(&BTreeSet::new(), &[], false).unwrap(); + assert!(!runtime.snapshot_hydration_cached(&snapshot).unwrap()); + } + + #[test] + fn local_commits_record_side_cache_hydration_proofs() { + let tokio_rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let remote = Arc::new(RemoteConfig::Memory.build().unwrap()); + let storage = Arc::new(FjallStorage::open_temporary().unwrap()); + let runtime = Runtime::new(tokio_rt.handle().clone(), remote, storage, None); + let vid = runtime.volume_open(None, None, None).unwrap().vid; + + let mut first = runtime.volume_writer(vid.clone()).unwrap(); + first + .write_page(PageIdx::must_new(1), Page::test_filled(1)) + .unwrap(); + first + .write_page(PageIdx::must_new(2), Page::test_filled(2)) + .unwrap(); + let first = first.commit().unwrap().snapshot().clone(); + assert!(runtime.snapshot_hydration_cached(&first).unwrap()); + + let mut second = runtime.volume_writer(vid).unwrap(); + second + .write_page(PageIdx::must_new(2), Page::test_filled(3)) + .unwrap(); + let second = second.commit().unwrap().snapshot().clone(); + assert!(runtime.snapshot_hydration_cached(&second).unwrap()); + } + #[test] fn runtime_sanity() { let tokio_rt = tokio::runtime::Builder::new_current_thread() diff --git a/packages/graft-remote-cloudflare/package.json b/packages/graft-remote-cloudflare/package.json index ef05db99..644789f4 100644 --- a/packages/graft-remote-cloudflare/package.json +++ b/packages/graft-remote-cloudflare/package.json @@ -1,6 +1,6 @@ { "name": "@eidos.space/graft-remote-cloudflare", - "version": "0.2.0", + "version": "0.2.1", "description": "Cloudflare storage and authentication adapters for @eidos.space/graft-remote", "type": "module", "license": "MIT", diff --git a/packages/graft-remote-hono/package.json b/packages/graft-remote-hono/package.json index 3ea65512..9709b4df 100644 --- a/packages/graft-remote-hono/package.json +++ b/packages/graft-remote-hono/package.json @@ -1,6 +1,6 @@ { "name": "@eidos.space/graft-remote-hono", - "version": "0.2.0", + "version": "0.2.1", "description": "Hono adapter for @eidos.space/graft-remote", "type": "module", "license": "MIT", diff --git a/packages/graft-remote/package.json b/packages/graft-remote/package.json index 4b78a4a0..0b5a506b 100644 --- a/packages/graft-remote/package.json +++ b/packages/graft-remote/package.json @@ -1,6 +1,6 @@ { "name": "@eidos.space/graft-remote", - "version": "0.2.0", + "version": "0.2.1", "description": "Framework-neutral types and protocol engine for Graft remote services", "type": "module", "license": "MIT", diff --git a/packages/graft-sdk/benchmark/PERFORMANCE.md b/packages/graft-sdk/benchmark/PERFORMANCE.md new file mode 100644 index 00000000..d9c90268 --- /dev/null +++ b/packages/graft-sdk/benchmark/PERFORMANCE.md @@ -0,0 +1,280 @@ +# Graft SDK performance report + +Date: 2026-08-02 + +Release candidate: `@eidos.space/graft` 0.3.5 +Baseline: published `@eidos.space/graft` 0.3.4 + +## Executive summary + +This release is aimed at the application workflow that exposed the bottleneck: opening Version +History and creating a checkpoint in an Eidos Space containing a 460.7 MB SQLite file with a +million-row table. It also adds resumable Remote uploads, but Remote transfer latency is kept out +of the local performance claims below. + +The result is not a blanket claim that every operation is faster. The release makes the expensive +SQLite inspection paths incremental and bounded while keeping ordinary Git-like repository work +within benchmark noise: + +- On the real 460.7 MB Eidos fixture, dirty status fell from 5.40 s to 19.7 ms, the working table + summary from 18.84 s to 1.53 s, and the selected metadata-table row page from 10.06 s to 1.69 ms. +- The same fixture's metadata-only checkpoint commit fell from 21.30 s to 1.69 ms. Initial + stage+commit fell from 10.03 s to 3.78 s. +- Peak RSS on that workflow fell from 3.88 GiB to 672.7 MiB. +- On the paired 5.6 MB Git-like workload, init, stage, commit, row diff, checkout, and filesystem + push stayed within the harness's noise band. No storage-byte amplification was introduced. +- A known scalability limit remains: `stagePaths` accepts a batch but currently executes each path + serially. Staging 1,000 changed paths in a 50,000-file worktree takes about 328 s. Applications + must not put this path on file-open or panel-open critical paths, and a true repository-level + batch implementation is the next Graft performance priority. + +## What changed + +The implementation follows the same broad separation that makes Git responsive, adapted to +SQLite rather than pretending a database is an opaque blob: + +1. **Cheap classification first.** Repository metadata and file fingerprints determine whether + cached status is reusable. Remote-tracking projection can refresh without throwing away a + proven local worktree classification. +2. **Content-addressed derived indexes.** Large SQLite files use checksummed page-hash indexes and + worktree probes under replaceable cache storage. Cache corruption or a racy fingerprint falls + back to authoritative comparison. +3. **Stable reads.** Rollback-journal databases are read under a shared SQLite transaction; WAL + databases use an online-backup snapshot. Repository reads never observe a torn database image. +4. **Stage prepares commit.** Staging retains the exact prepared SQLite snapshot and changed-table + candidates. Commit consumes that canonical staged image instead of re-reading the live + worktree. +5. **Layout-specific diff algorithms.** Ordinary rowid tables use the existing changed-page-aware + diff for checkpoint summaries. Eidos `WITHOUT ROWID` tables use bounded primary-key and + changed-page traversal. A single algorithm was measurably wrong for one of these layouts. +6. **Hydration proofs stay off the database write set.** Exact snapshot-presence proofs are atomic, + content-addressed side-cache files. An earlier prototype used an extra Fjall keyspace and made + ordinary stage/commit 10–20% slower; the paired gate caught it and that design was removed. + +This corresponds to Git's index/stat cache, commit-graph, and replaceable derived metadata, but +the cache contents and invalidation rules are Graft-specific. Authoritative snapshots and commits +remain unchanged. + +## Test environment and methodology + +| Item | Value | +| --- | --- | +| Host | Apple M2, 8 logical CPUs, 24 GiB RAM | +| OS | Darwin 24.6.0, arm64 | +| Local benchmark runtime | Node.js 26.0.0 | +| Release compatibility gate | Node.js 20 and 24 on five native targets | +| Timing | Wall clock via `performance.now()` or paired Rust harness | +| Isolation | Fresh repository and fresh Node child per scale point | +| Fixture generation | Timed separately and excluded from Graft operation timings | +| Git-like comparison | 4 aligned baseline/candidate pairs after 1 warm-up pair | +| Remote used for core comparison | Deterministic filesystem Remote | + +Node.js 26 is the locally installed benchmark runtime, not a new support claim. The release +workflow separately builds and runs the SDK contract on Node.js 20 and 24 for macOS arm64/x64, +Linux glibc arm64/x64, and Windows x64. + +The synthetic matrix covers: + +- 100, 1,000, 10,000, and 50,000 ordinary files, each 256 bytes; +- 1, 100, and 1,000 changed-path rounds where the scale permits; +- SQLite databases with 10,000, 100,000, and 1,000,000 rows; +- 0.75 MB, 27.4 MB, and 410.6 MB SQLite file sizes; +- 1, 100, and 10,000 changed-row rounds; +- cold open/init/stage/commit, resident and reopened status, working and historical summaries, + first 100 changed rows, incremental stage/commit, history, RSS, and repository bytes. + +## Git-like end-to-end workflow + +The paired dataset contains a 5.6 MB SQLite database with 20,000 rows, 64 text files, two binary +files, a 10% row update, checkout, and push to a filesystem Remote. + +| Operation | 0.3.4 | 0.3.5 candidate | Paired change | +| --- | ---: | ---: | ---: | +| Repository init | 4.22 ms | 4.19 ms | -2.6% | +| Initial stage | 955.88 ms | 977.75 ms | +4.2% (noisy) | +| Initial commit | 520.48 ms | 482.13 ms | -4.0% | +| Stage 10% row update | 539.73 ms | 525.38 ms | -1.5% | +| Incremental commit | 501.34 ms | 541.47 ms | +5.5% | +| Row diff | 557.77 ms | 613.24 ms | +9.5% (high variance) | +| Checkout parent | 589.10 ms | 544.21 ms | -9.2% | +| Filesystem Remote push | 646.71 ms | 657.29 ms | +2.5% | + +The comparison marks only incremental commit as a statistically visible small regression. It is +about 40 ms on this dataset and is outweighed in the target Eidos checkpoint path by the prepared +SQLite stage described below. The full aligned samples and median absolute deviations are in +[`results/git-workflow-comparison.md`](results/git-workflow-comparison.md). + +Storage was unchanged to the displayed precision: the 9.85 MiB worktree produced a 9.66 MiB +repository after the initial commit and 16.97 MiB after two commits in both versions. The candidate +adds two tiny replaceable cache files in this run; repository objects, snapshot bytes, external +payloads, Remote bytes, and object counts are otherwise unchanged. + +## Ordinary-file scale + +| Files | Initial stage | Initial commit | Hot clean status p50 | Reopened status | Peak RSS | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 100 | 48.6 ms | 3.45 ms | 1.88 ms | 13.0 ms | 64.7 MiB | +| 1,000 | 216.0 ms | 12.3 ms | 13.1 ms | 59.1 ms | 166.0 MiB | +| 10,000 | 2.17 s | 133 ms | 138 ms | 333 ms | 286.6 MiB | +| 50,000 | 14.82 s | 688 ms | 723 ms | 1.60 s | 680.2 MiB | + +The candidate's 50,000-file initial stage is about 12% faster than 0.3.4. Status remains O(number +of visible paths) because Graft must validate fingerprints; it is designed to run after the local +editor is interactive, not before a file can open. + +### Explicit path batch limit + +| Worktree | 1 changed path | 100 changed paths | 1,000 changed paths | +| ---: | ---: | ---: | ---: | +| 1,000 files | 4.16 ms | 607 ms | 13.68 s | +| 10,000 files | 52.0 ms | 5.84 s | 66.89 s | +| 50,000 files | 268 ms | 31.78 s | 327.96 s | + +This is the clearest remaining Graft API problem. The JavaScript call is batched, but the core +implementation still repeats repository work per path. A future `stage_pathset` primitive should +resolve ignore state, index changes, and one repository transaction for the whole set. + +## Synthetic SQLite scale + +| Rows | SQLite size | Initial stage | Initial commit | Peak RSS | +| ---: | ---: | ---: | ---: | ---: | +| 10,000 | 0.75 MB | 11.7 ms | 8.60 ms | 79.0 MiB | +| 100,000 | 27.4 MB | 86.5 ms | 55.5 ms | 228.5 MiB | +| 1,000,000 | 410.6 MB | 1.10 s | 2.26 s | 770.8 MiB | + +For the 410.6 MB database, compared with 0.3.4: + +- working summaries improved from 4.89–5.31 s to 3.02–3.65 s; +- the first 100 changed rows improved from 3.66–5.10 s to 0.44–1.73 s; +- incremental stage improved from 1.92–2.16 s to 1.09–1.50 s; +- incremental commit remained 1.23–1.50 s versus 1.27–1.58 s; +- historical summaries improved from 5.18–6.00 s to 3.92–4.09 s. + +The candidate uses about 16% more peak RSS on this synthetic rowid fixture (770.8 MiB versus +662.3 MiB) because the faster checkpoint summary decodes rows on changed pages. This is bounded by +changed-page payload, not total-table payload. The real Eidos layout takes the primary-key path and +shows the opposite memory result. + +Persistent page-hash caching is enabled only for SQLite files of at least 16 MiB. Below that size, +the fixed cache/index cost was larger than an authoritative scan; small databases therefore bypass +it and keep the simpler path. + +## Real Eidos Space fixture + +The application fixture is an anonymized `Untitled.eidos` test file, copied read-only into a +temporary repository for each run. Both compared runs used the same 460,689,408-byte source. The +benchmark mutates only `eidos__meta` in the temporary copy. + +| Operation | 0.3.4 | 0.3.5 candidate | Change | +| --- | ---: | ---: | ---: | +| Session open | 464 ms | 501 ms | +8.0% | +| Repository init | 445 ms | 410 ms | -7.8% | +| Initial stage | 6.37 s | 1.80 s | -71.7% | +| Initial commit | 3.66 s | 1.98 s | -46.0% | +| Dirty status | 5.40 s | 19.7 ms | -99.6% | +| Working table summary | 18.84 s | 1.53 s | -91.9% | +| First metadata-table row page | 10.06 s | 1.69 ms | -99.98% | +| Stage metadata change | 6.05 s | 1.58 s | -73.9% | +| Commit metadata change | 21.30 s | 1.69 ms | -99.99% | +| Post-commit status | 7.45 s | 771 ms | -89.7% | +| Historical summary | 12.53 s | 4.92 s | -60.7% | +| Peak RSS | 3.88 GiB | 672.7 MiB | -83.1% | + +Session open remains roughly 0.4–0.5 s because it opens the runtime and local stores. Eidos must +therefore keep it off the editor's critical path: local SQLite becomes editable first, Version +History attaches later, and Remote state is requested only when Sync is enabled. + +Historical summary is substantially better but still takes seconds on this fixture. It belongs in +a cancellable background request with cached summary rendering; it must not block table switching. + +## Remote and Cloudflare scope + +The core paired test uses a filesystem Remote to isolate object creation, bundling, ref publication, +and local repository costs from public-network noise. The Remote changes in this release add: + +- resumable multipart segment upload negotiation; +- immutable part retries and completion reconciliation; +- Cloudflare R2 multipart storage support; +- compatibility fallback when a Remote does not advertise multipart capabilities. + +Protocol and Cloudflare worker tests cover retry, duplicate parts, completion, and fallback. This +report intentionally does not present one staging-network latency sample as a universal Remote +performance number. A separate controlled Cloudflare load test should vary segment size, +concurrency, packet loss, RTT, and interrupted uploads before setting production transfer defaults. + +## Application integration rules + +The benchmark results imply a strict priority order for Eidos: + +1. Open and edit the local Eidos file. +2. Attach the retained Graft repository session in the background. +3. Show cached Version/Sync state immediately, then refresh local status. +4. Load summaries before row payloads; load rows only for the selected table. +5. If Sync is enabled, refresh Remote projection after local state is usable. +6. Cancel stale panel/table requests without surfacing cancellation as an error. + +No status, history, Remote, entitlement, quota, or checkpoint request is allowed to gate local +file opening. This is an application scheduling contract in addition to a Graft performance +property. + +## Reproduction and raw data + +Build the local SDK and run the full matrix: + +```sh +pnpm --dir packages/graft-sdk build:native +GRAFT_PERF_PROFILE=full GRAFT_PERF_ITERATIONS=5 \ + GRAFT_PERF_OUTPUT=benchmark/results/performance-matrix-candidate-macos-arm64.json \ + pnpm --dir packages/graft-sdk bench:matrix +``` + +Run the real Eidos fixture: + +```sh +GRAFT_REAL_EIDOS_SOURCE=/absolute/path/to/Untitled.eidos \ + GRAFT_PERF_OUTPUT=benchmark/results/real-eidos-candidate-macos-arm64.json \ + pnpm --dir packages/graft-sdk bench:real-eidos +``` + +Run the deterministic paired core benchmark with two release binaries: + +```sh +cargo build --release --locked -p graft-tool -p graft-bench +./target/release/graft-bench run-paired \ + --baseline-graft-bin /path/to/0.3.4/graft \ + --candidate-graft-bin ./target/release/graft \ + --baseline-output packages/graft-sdk/benchmark/results/git-workflow-v0.3.4.json \ + --candidate-output packages/graft-sdk/benchmark/results/git-workflow-candidate.json \ + --baseline-label graft-sdk-v0.3.4 --candidate-label graft-sdk-v0.3.5 \ + --profile ci --samples 4 --warmups 1 +``` + +Raw results: + +- [`performance-matrix-v0.3.4-macos-arm64.json`](results/performance-matrix-v0.3.4-macos-arm64.json) +- [`performance-matrix-candidate-macos-arm64.json`](results/performance-matrix-candidate-macos-arm64.json) +- [`real-eidos-v0.3.4-macos-arm64.json`](results/real-eidos-v0.3.4-macos-arm64.json) +- [`real-eidos-candidate-macos-arm64.json`](results/real-eidos-candidate-macos-arm64.json) +- [`git-workflow-v0.3.4.json`](results/git-workflow-v0.3.4.json) +- [`git-workflow-candidate.json`](results/git-workflow-candidate.json) + +## Release gates and next work + +This release is acceptable when: + +- SDK and SQLite tests pass on the full workspace; +- the paired Git-like workflow has no broad regression; +- the real Eidos fixture keeps dirty status under 100 ms, selected metadata rows under 100 ms, + and peak RSS under 1 GiB on the reference host; +- all published native packages pass Node.js 20/24 contract tests; +- Remote multipart packages pass typecheck, protocol tests, packaging verification, and a + Cloudflare dry-run deployment. + +Next priorities, in order: + +1. implement true core-level batch staging instead of a loop around atomic path operations; +2. persist enough safe table/page change metadata to reduce historical summary below one second; +3. add Linux x64/arm64 benchmark runners and a Windows smoke profile; +4. add controlled Cloudflare multipart throughput and interruption benchmarks; +5. make the benchmark harness record an explicit candidate artifact digest in addition to the + harness checkout revision. diff --git a/packages/graft-sdk/benchmark/performance-matrix.mjs b/packages/graft-sdk/benchmark/performance-matrix.mjs new file mode 100644 index 00000000..66e75aa4 --- /dev/null +++ b/packages/graft-sdk/benchmark/performance-matrix.mjs @@ -0,0 +1,448 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { performance } from "node:perf_hooks" +import { createRequire } from "node:module" +import { fileURLToPath } from "node:url" + +const require = createRequire(import.meta.url) +const { DatabaseSync } = require("node:sqlite") +const sdkModule = process.env.GRAFT_SDK_MODULE ?? ".." +const sdkRoot = path.resolve(import.meta.dirname, sdkModule) +const sdkVersion = require(path.join(sdkRoot, "package.json")).version +const { RepositorySession } = require(sdkRoot) +const benchmarkFile = fileURLToPath(import.meta.url) +const profile = process.env.GRAFT_PERF_PROFILE ?? "full" +const iterations = positiveInteger(process.env.GRAFT_PERF_ITERATIONS, 5) + +if (process.argv[2] === "--child") { + const kind = process.argv[3] + const scenario = JSON.parse(process.argv[4]) + const result = + kind === "files" + ? await runFileScenario(scenario) + : await runSqliteScenario(scenario) + process.stdout.write(`${JSON.stringify(result)}\n`) + process.exit(0) +} + +const output = path.resolve( + process.env.GRAFT_PERF_OUTPUT ?? + path.join(import.meta.dirname, "results", "performance-matrix-macos-arm64.json") +) +const selected = profiles(profile) +const started = performance.now() +const fileScenarios = [] +const sqliteScenarios = [] + +for (const scenario of selected.files) { + process.stderr.write( + `files: ${scenario.file_count.toLocaleString()} paths (${scenario.change_counts.join(", ")} changed)\n` + ) + fileScenarios.push(await runChild("files", scenario)) +} +for (const scenario of selected.sqlite) { + process.stderr.write( + `sqlite: ${scenario.rows.toLocaleString()} rows, ${scenario.payload_bytes} byte payloads\n` + ) + sqliteScenarios.push(await runChild("sqlite", scenario)) +} + +const report = { + schema: "graft-sdk-performance-matrix-v1", + generated_at: new Date().toISOString(), + source_revision: await gitRevision(), + sdk: { module: sdkRoot, version: sdkVersion }, + profile, + methodology: { + iterations, + timing: "wall-clock milliseconds via performance.now()", + percentiles: "nearest-rank over repeated resident-session reads", + fixture_generation: "reported separately and excluded from Graft operation timings", + isolation: "one fresh repository and Node.js child process per scale point", + }, + environment: { + platform: process.platform, + arch: process.arch, + node: process.version, + cpu: os.cpus()[0]?.model ?? "unknown", + cpu_count: os.cpus().length, + memory_bytes: os.totalmem(), + os: `${os.type()} ${os.release()}`, + }, + total_wall_milliseconds: round(performance.now() - started), + files: fileScenarios, + sqlite: sqliteScenarios, +} +await fs.mkdir(path.dirname(output), { recursive: true }) +await fs.writeFile(output, `${JSON.stringify(report, null, 2)}\n`) +process.stderr.write(`wrote ${output}\n`) + +function profiles(name) { + if (name === "smoke") { + return { + files: [{ file_count: 100, file_bytes: 256, change_counts: [1, 10] }], + sqlite: [{ rows: 1_000, payload_bytes: 64, change_counts: [1, 100] }], + } + } + if (name !== "full") throw new Error(`unknown profile: ${name}`) + return { + files: [ + { file_count: 100, file_bytes: 256, change_counts: [1, 10, 100] }, + { file_count: 1_000, file_bytes: 256, change_counts: [1, 100, 1_000] }, + { file_count: 10_000, file_bytes: 256, change_counts: [1, 100, 1_000] }, + { file_count: 50_000, file_bytes: 256, change_counts: [1, 100, 1_000] }, + ], + sqlite: [ + { rows: 10_000, payload_bytes: 64, change_counts: [1, 100, 10_000] }, + { rows: 100_000, payload_bytes: 256, change_counts: [1, 100, 10_000] }, + { rows: 1_000_000, payload_bytes: 384, change_counts: [1, 100, 10_000] }, + ], + } +} + +async function runFileScenario(scenario) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "graft-perf-files-")) + try { + const fixture = await timed(() => createFiles(root, scenario)) + const opened = await timed(() => RepositorySession.open(root)) + const session = opened.value + try { + const init = await timed(() => session.init()) + const stageInitial = await timed(() => session.addAll()) + const initialCommit = await timed(() => session.commit("initial files")) + const hotCleanStatus = await sample(iterations, () => session.statusIncremental()) + const rounds = [] + for (let round = 0; round < scenario.change_counts.length; round += 1) { + const changed = Math.min(scenario.change_counts[round], scenario.file_count) + const paths = filePaths(changed) + await mutateFiles(root, paths, round) + const dirtyStatus = await timed(() => session.statusIncremental()) + assert.equal(dirtyStatus.value.status.dirty, true) + const explicitDiff = await timed(() => + session.diffPaths({ paths: paths.slice(0, 100), limit: 100 }) + ) + const stagePaths = await timed(() => session.stagePaths({ paths })) + const commit = await timed(() => session.commit(`change ${changed} files`)) + const postCommitStatus = await timed(() => session.statusIncremental()) + assert.equal(postCommitStatus.value.status.dirty, false) + rounds.push({ + changed_paths: changed, + status_dirty: metric(dirtyStatus), + explicit_path_diff: metric(explicitDiff), + stage_paths: metric(stagePaths), + commit: metric(commit), + status_after_commit: metric(postCommitStatus), + status_after_commit_cache_hit: + postCommitStatus.value.telemetry.status_cache_hit, + }) + } + const history = await sample(iterations, () => + session.historySummaries({ limit: 50 }) + ) + await session.close() + const reopened = await timed(() => RepositorySession.open(root)) + const reopenStatus = await timed(() => reopened.value.statusIncremental()) + await reopened.value.close() + return { + fixture: scenario, + fixture_generation: metric(fixture), + repository_bytes: await directoryBytes(root), + peak_rss_bytes: peakRssBytes(), + operations: { + session_open: metric(opened), + init: metric(init), + stage_initial: metric(stageInitial), + commit_initial: metric(initialCommit), + clean_status_hot: sampledMetric(hotCleanStatus), + history_summaries_50: sampledMetric(history), + reopen: metric(reopened), + status_after_reopen: metric(reopenStatus), + persistent_status_hit: + reopenStatus.value.telemetry.persistent_snapshot_hit, + }, + mutation_rounds: rounds, + } + } finally { + await session.close().catch(() => undefined) + } + } finally { + await fs.rm(root, { recursive: true, force: true }) + } +} + +async function runSqliteScenario(scenario) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "graft-perf-sqlite-")) + try { + const databasePath = path.join(root, "space.eidos") + const fixture = await timed(() => createDatabase(databasePath, scenario)) + const databaseBytes = (await fs.stat(databasePath)).size + const opened = await timed(() => RepositorySession.open(root)) + const session = opened.value + try { + const init = await timed(() => session.init()) + const stageInitial = await timed(() => session.addAll()) + const initialCommit = await timed(() => session.commit("initial database")) + let priorCommit = commitId(initialCommit.value) + const hotCleanStatus = await sample(iterations, () => session.statusIncremental()) + const rounds = [] + for (let round = 0; round < scenario.change_counts.length; round += 1) { + const changed = Math.min(scenario.change_counts[round], scenario.rows) + await updateRows(databasePath, scenario.rows, changed, round) + const dirtyStatus = await timed(() => session.statusIncremental()) + assert.equal(dirtyStatus.value.status.dirty, true) + const workingSummary = await timed(() => + session.diffSqlitePaths({ paths: ["space.eidos"], mode: "summary" }) + ) + const workingRows = await timed(() => + session.diffSqlitePaths({ + paths: ["space.eidos"], + mode: "rows", + table: "records", + rowLimit: 100, + }) + ) + const stage = await timed(() => + session.stagePaths({ paths: ["space.eidos"] }) + ) + const commit = await timed(() => session.commit(`change ${changed} rows`)) + const nextCommit = commitId(commit.value) + const postCommitStatus = await timed(() => session.statusIncremental()) + assert.equal(postCommitStatus.value.status.dirty, false) + const historicalSummary = await timed(() => + session.diffSqlitePaths({ + paths: ["space.eidos"], + from: priorCommit, + to: nextCommit, + mode: "summary", + }) + ) + rounds.push({ + changed_rows: changed, + status_dirty: metric(dirtyStatus), + working_summary: metric(workingSummary), + working_rows_first_100: metric(workingRows), + stage: metric(stage), + commit: metric(commit), + status_after_commit: metric(postCommitStatus), + status_after_commit_cache_hit: + postCommitStatus.value.telemetry.status_cache_hit, + historical_summary: metric(historicalSummary), + }) + priorCommit = nextCommit + } + const history = await sample(iterations, () => + session.historySummaries({ limit: 50 }) + ) + await session.close() + const reopened = await timed(() => RepositorySession.open(root)) + const reopenStatus = await timed(() => reopened.value.statusIncremental()) + await reopened.value.close() + return { + fixture: { ...scenario, database_bytes: databaseBytes }, + fixture_generation: metric(fixture), + repository_bytes: await directoryBytes(root), + peak_rss_bytes: peakRssBytes(), + operations: { + session_open: metric(opened), + init: metric(init), + stage_initial: metric(stageInitial), + commit_initial: metric(initialCommit), + clean_status_hot: sampledMetric(hotCleanStatus), + history_summaries_50: sampledMetric(history), + reopen: metric(reopened), + status_after_reopen: metric(reopenStatus), + persistent_status_hit: + reopenStatus.value.telemetry.persistent_snapshot_hit, + }, + mutation_rounds: rounds, + } + } finally { + await session.close().catch(() => undefined) + } + } finally { + await fs.rm(root, { recursive: true, force: true }) + } +} + +async function createFiles(root, scenario) { + const pending = [] + for (let index = 0; index < scenario.file_count; index += 1) { + const file = path.join(root, filePath(index)) + pending.push( + fs.mkdir(path.dirname(file), { recursive: true }).then(() => + fs.writeFile(file, `${index}:`.padEnd(scenario.file_bytes, "x")) + ) + ) + if (pending.length === 512) { + await Promise.all(pending) + pending.length = 0 + } + } + await Promise.all(pending) +} + +function filePaths(count) { + return Array.from({ length: count }, (_, index) => filePath(index)) +} + +function filePath(index) { + const directory = Math.floor(index / 1_000).toString().padStart(4, "0") + return path.join("files", directory, `item-${index.toString().padStart(6, "0")}.txt`) +} + +async function mutateFiles(root, paths, round) { + for (let index = 0; index < paths.length; index += 256) { + await Promise.all( + paths.slice(index, index + 256).map((relative, offset) => + fs.appendFile(path.join(root, relative), `\n${round}:${index + offset}`) + ) + ) + } +} + +function createDatabase(databasePath, scenario) { + const database = new DatabaseSync(databasePath) + try { + database.exec( + "PRAGMA journal_mode=DELETE; PRAGMA synchronous=OFF; " + + "CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 0)" + ) + const insert = database.prepare( + "INSERT INTO records (id, value) VALUES (?, ?)" + ) + const prefix = "x".repeat(Math.max(1, scenario.payload_bytes - 16)) + database.exec("BEGIN") + for (let index = 1; index <= scenario.rows; index += 1) { + insert.run(index, `${prefix}${index.toString(16).padStart(16, "0")}`) + } + database.exec("COMMIT; PRAGMA optimize") + } finally { + database.close() + } +} + +function updateRows(databasePath, totalRows, count, round) { + const database = new DatabaseSync(databasePath) + try { + const update = database.prepare( + "UPDATE records SET revision = ?, value = value || ? WHERE id = ?" + ) + database.exec("BEGIN") + const stride = Math.max(1, Math.floor(totalRows / count)) + for (let index = 0; index < count; index += 1) { + update.run(round + 1, `:${round}`, Math.min(totalRows, index * stride + 1)) + } + database.exec("COMMIT") + } finally { + database.close() + } +} + +function commitId(result) { + const id = result?.commit?.id + assert.equal(typeof id, "string") + return id +} + +async function runChild(kind, scenario) { + const childStarted = performance.now() + const child = spawn( + process.execPath, + [benchmarkFile, "--child", kind, JSON.stringify(scenario)], + { stdio: ["ignore", "pipe", "inherit"] } + ) + const chunks = [] + child.stdout.on("data", (chunk) => chunks.push(chunk)) + const code = await new Promise((resolve) => child.on("close", resolve)) + if (code !== 0) throw new Error(`${kind} benchmark child exited with ${code}`) + const result = JSON.parse(Buffer.concat(chunks).toString("utf8")) + result.scenario_wall_milliseconds = round(performance.now() - childStarted) + return result +} + +async function gitRevision() { + const child = spawn("git", ["rev-parse", "HEAD"], { + cwd: path.resolve(import.meta.dirname, "../../.."), + stdio: ["ignore", "pipe", "ignore"], + }) + const chunks = [] + child.stdout.on("data", (chunk) => chunks.push(chunk)) + const code = await new Promise((resolve) => child.on("close", resolve)) + return code === 0 ? Buffer.concat(chunks).toString("utf8").trim() : "unknown" +} + +async function timed(operation) { + const started = performance.now() + const value = await operation() + return { milliseconds: performance.now() - started, value } +} + +async function sample(count, operation) { + const milliseconds = [] + const values = [] + for (let index = 0; index < count; index += 1) { + const measured = await timed(operation) + milliseconds.push(measured.milliseconds) + values.push(measured.value) + } + return { milliseconds, values } +} + +function metric(measured) { + return { + milliseconds: round(measured.milliseconds), + response_bytes: jsonBytes(measured.value), + } +} + +function sampledMetric(measured) { + const sorted = measured.milliseconds.toSorted((left, right) => left - right) + return { + samples: measured.milliseconds.map(round), + min: round(sorted[0]), + p50: round(percentile(sorted, 0.5)), + p95: round(percentile(sorted, 0.95)), + max: round(sorted.at(-1)), + response_bytes: jsonBytes(measured.values.at(-1)), + } +} + +function percentile(sorted, quantile) { + return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * quantile))] +} + +function jsonBytes(value) { + const encoded = JSON.stringify(value) + return encoded === undefined ? 0 : Buffer.byteLength(encoded) +} + +function peakRssBytes() { + return process.resourceUsage().maxRSS * 1024 +} + +async function directoryBytes(root) { + let total = 0 + for (const entry of await fs.readdir(root, { withFileTypes: true })) { + const target = path.join(root, entry.name) + total += entry.isDirectory() + ? await directoryBytes(target) + : (await fs.stat(target)).size + } + return total +} + +function positiveInteger(value, fallback) { + if (value === undefined) return fallback + const parsed = Number.parseInt(value, 10) + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`expected a positive integer, received ${value}`) + } + return parsed +} + +function round(value) { + return Math.round(value * 1_000) / 1_000 +} diff --git a/packages/graft-sdk/benchmark/real-eidos.mjs b/packages/graft-sdk/benchmark/real-eidos.mjs new file mode 100644 index 00000000..ff51e242 --- /dev/null +++ b/packages/graft-sdk/benchmark/real-eidos.mjs @@ -0,0 +1,137 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { performance } from "node:perf_hooks" +import { createRequire } from "node:module" + +const require = createRequire(import.meta.url) +const { DatabaseSync } = require("node:sqlite") +const configuredSource = process.env.GRAFT_REAL_EIDOS_SOURCE +if (!configuredSource) { + throw new Error("GRAFT_REAL_EIDOS_SOURCE must point to an Eidos fixture") +} +const source = path.resolve(configuredSource) +const sdkRoot = path.resolve( + import.meta.dirname, + process.env.GRAFT_SDK_MODULE ?? ".." +) +const sdkLabel = + process.env.GRAFT_SDK_LABEL ?? + (process.env.GRAFT_SDK_MODULE ? "external-sdk" : "workspace-sdk") +const { RepositorySession } = require(sdkRoot) +const sdkVersion = require(path.join(sdkRoot, "package.json")).version +const root = await fs.mkdtemp(path.join(os.tmpdir(), "graft-real-eidos-")) + +try { + const target = path.join(root, "Untitled.eidos") + const copy = await timed(() => fs.copyFile(source, target)) + const sourceBytes = (await fs.stat(target)).size + const opened = await timed(() => RepositorySession.open(root)) + const session = opened.value + try { + const init = await timed(() => session.init()) + const stageInitial = await timed(() => session.addAll()) + const initialCommit = await timed(() => session.commit("real Eidos baseline")) + const baselineCommit = commitId(initialCommit.value) + mutateMeta(target) + const dirtyStatus = await timed(() => session.statusIncremental()) + const summary = await timed(() => + session.diffSqlitePaths({ paths: ["Untitled.eidos"], mode: "summary" }) + ) + const rows = await timed(() => + session.diffSqlitePaths({ + paths: ["Untitled.eidos"], + mode: "rows", + table: "eidos__meta", + rowLimit: 100, + }) + ) + const stage = await timed(() => + session.stagePaths({ paths: ["Untitled.eidos"] }) + ) + const commit = await timed(() => session.commit("update metadata")) + const updatedCommit = commitId(commit.value) + const postCommitStatus = await timed(() => session.statusIncremental()) + const historicalSummary = await timed(() => + session.diffSqlitePaths({ + paths: ["Untitled.eidos"], + from: baselineCommit, + to: updatedCommit, + mode: "summary", + }) + ) + const history = await timed(() => session.historySummaries({ limit: 50 })) + const report = { + schema: "graft-sdk-real-eidos-benchmark-v1", + generated_at: new Date().toISOString(), + sdk: { module: sdkLabel, version: sdkVersion }, + source: { path: `fixture:${path.basename(source)}`, bytes: sourceBytes }, + environment: { + platform: process.platform, + arch: process.arch, + node: process.version, + cpu: os.cpus()[0]?.model ?? "unknown", + memory_bytes: os.totalmem(), + }, + fixture_copy: metric(copy), + peak_rss_bytes: process.resourceUsage().maxRSS * 1024, + operations: { + session_open: metric(opened), + init: metric(init), + stage_initial: metric(stageInitial), + commit_initial: metric(initialCommit), + status_dirty: metric(dirtyStatus), + working_summary: metric(summary), + working_meta_rows: metric(rows), + stage_meta_change: metric(stage), + commit_meta_change: metric(commit), + status_after_commit: metric(postCommitStatus), + historical_summary: metric(historicalSummary), + history_summaries_50: metric(history), + }, + } + const output = process.env.GRAFT_PERF_OUTPUT + if (output) { + await fs.mkdir(path.dirname(path.resolve(output)), { recursive: true }) + await fs.writeFile(path.resolve(output), `${JSON.stringify(report, null, 2)}\n`) + } else { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`) + } + } finally { + await session.close().catch(() => undefined) + } +} finally { + await fs.rm(root, { recursive: true, force: true }) +} + +function mutateMeta(databasePath) { + const database = new DatabaseSync(databasePath) + try { + database.exec( + "UPDATE eidos__meta SET revision = revision + 1, updated_at = '2026-08-02T12:00:00.000Z' WHERE singleton = 1" + ) + } finally { + database.close() + } +} + +function commitId(result) { + const id = result?.commit?.id + assert.equal(typeof id, "string") + return id +} + +async function timed(operation) { + const started = performance.now() + const value = await operation() + return { milliseconds: performance.now() - started, value } +} + +function metric(measured) { + const encoded = JSON.stringify(measured.value) + return { + milliseconds: Math.round(measured.milliseconds * 1_000) / 1_000, + response_bytes: encoded === undefined ? 0 : Buffer.byteLength(encoded), + } +} diff --git a/packages/graft-sdk/benchmark/results/git-workflow-candidate.json b/packages/graft-sdk/benchmark/results/git-workflow-candidate.json new file mode 100644 index 00000000..45c61896 --- /dev/null +++ b/packages/graft-sdk/benchmark/results/git-workflow-candidate.json @@ -0,0 +1,454 @@ +{ + "schema_version": 2, + "label": "next", + "graft_version": "graft-tool 0.11.0", + "provenance": { + "run_id": "18c7e773140911c8-a324", + "report_kind": "paired_candidate", + "harness_label": "next", + "build_profile": "release", + "os": "macos", + "arch": "aarch64", + "runner_image": null, + "pair_order": [ + "baseline_first", + "candidate_first", + "baseline_first", + "candidate_first" + ] + }, + "parameters": { + "profile": "ci", + "sqlite_rows": 20000, + "updated_rows": 2000, + "row_payload_bytes": 256, + "text_file_count": 64, + "text_file_bytes": 4096, + "binary_file_count": 2, + "binary_file_bytes": 2097152 + }, + "sample_count": 4, + "warmup_count": 1, + "metrics": [ + { + "name": "speed.repo_init", + "display_name": "Repository init", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 4.1942710000000005, + "median_absolute_deviation": 0.061228999999999534, + "samples": [ + 4.222917, + 4.288082999999999, + 4.047292, + 4.165625 + ] + }, + { + "name": "speed.stage_initial", + "display_name": "Stage initial dataset", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 977.7455415, + "median_absolute_deviation": 37.54999950000001, + "samples": [ + 944.661167, + 935.729917, + 2907.912958, + 1010.829916 + ] + }, + { + "name": "speed.commit_initial", + "display_name": "Commit initial dataset", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 482.13347899999997, + "median_absolute_deviation": 4.83156249999999, + "samples": [ + 478.248792, + 486.01816599999995, + 476.35504099999997, + 547.7310829999999 + ] + }, + { + "name": "speed.stage_incremental", + "display_name": "Stage 10% row update", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 525.3829165, + "median_absolute_deviation": 15.681291500000043, + "samples": [ + 503.651667, + 515.751583, + 577.669625, + 535.0142500000001 + ] + }, + { + "name": "speed.commit_incremental", + "display_name": "Commit incremental update", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 541.4718545, + "median_absolute_deviation": 17.324250000000063, + "samples": [ + 551.037292, + 531.9064169999999, + 508.49750000000006, + 566.554917 + ] + }, + { + "name": "speed.row_diff", + "display_name": "Row diff between commits", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 613.2396665, + "median_absolute_deviation": 45.51516700000002, + "samples": [ + 671.445333, + 556.883791, + 578.565208, + 647.914125 + ] + }, + { + "name": "speed.checkout_parent", + "display_name": "Checkout parent revision", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 544.2087710000001, + "median_absolute_deviation": 9.482124999999996, + "samples": [ + 526.5662920000001, + 569.0277080000001, + 542.8870000000001, + 545.530542 + ] + }, + { + "name": "speed.push_fs_remote", + "display_name": "Push to filesystem remote", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 657.2866245, + "median_absolute_deviation": 23.889478999999994, + "samples": [ + 640.324333, + 626.469958, + 740.2082909999999, + 674.248916 + ] + }, + { + "name": "storage.worktree_bytes", + "display_name": "Worktree dataset", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10330112.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10330112.0, + 10330112.0, + 10330112.0, + 10330112.0 + ] + }, + { + "name": "storage.sqlite_bytes", + "display_name": "Materialized SQLite database", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 5873664.0, + "median_absolute_deviation": 0.0, + "samples": [ + 5873664.0, + 5873664.0, + 5873664.0, + 5873664.0 + ] + }, + { + "name": "storage.graft_initial_bytes", + "display_name": ".graft after initial commit", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10129093.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10129093.0, + 10129093.0, + 10129093.0, + 10129093.0 + ] + }, + { + "name": "storage.graft_incremental_bytes", + "display_name": ".graft after incremental commit", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 17799518.0, + "median_absolute_deviation": 0.0, + "samples": [ + 17799518.0, + 17799518.0, + 17799518.0, + 17799518.0 + ] + }, + { + "name": "storage.incremental_growth_bytes", + "display_name": "Incremental history growth", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 7670425.0, + "median_absolute_deviation": 0.0, + "samples": [ + 7670425.0, + 7670425.0, + 7670425.0, + 7670425.0 + ] + }, + { + "name": "storage.initial_amplification", + "display_name": "Initial storage amplification", + "group": "storage", + "unit": "ratio", + "lower_is_better": true, + "median": 0.9805404820393041, + "median_absolute_deviation": 0.0, + "samples": [ + 0.9805404820393041, + 0.9805404820393041, + 0.9805404820393041, + 0.9805404820393041 + ] + }, + { + "name": "storage.incremental_amplification", + "display_name": "Two-commit storage amplification", + "group": "storage", + "unit": "ratio", + "lower_is_better": true, + "median": 1.7230711535363799, + "median_absolute_deviation": 0.0, + "samples": [ + 1.7230711535363799, + 1.7230711535363799, + 1.7230711535363799, + 1.7230711535363799 + ] + }, + { + "name": "storage.fjall_incremental_bytes", + "display_name": "SQLite snapshot store", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 11132128.0, + "median_absolute_deviation": 0.0, + "samples": [ + 11132128.0, + 11132128.0, + 11132128.0, + 11132128.0 + ] + }, + { + "name": "storage.objects_incremental_bytes", + "display_name": "Repository objects", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 375129.0, + "median_absolute_deviation": 0.0, + "samples": [ + 375129.0, + 375129.0, + 375129.0, + 375129.0 + ] + }, + { + "name": "storage.payloads_incremental_bytes", + "display_name": "External file payloads", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 6291456.0, + "median_absolute_deviation": 0.0, + "samples": [ + 6291456.0, + 6291456.0, + 6291456.0, + 6291456.0 + ] + }, + { + "name": "storage.metadata_incremental_bytes", + "display_name": "Refs, index, and metadata", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 805.0, + "median_absolute_deviation": 0.0, + "samples": [ + 805.0, + 805.0, + 805.0, + 805.0 + ] + }, + { + "name": "storage.graft_file_count", + "display_name": ".graft file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 103.0, + "median_absolute_deviation": 0.0, + "samples": [ + 103.0, + 103.0, + 103.0, + 103.0 + ] + }, + { + "name": "storage.objects_file_count", + "display_name": "Repository object file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 74.0, + "median_absolute_deviation": 0.0, + "samples": [ + 74.0, + 74.0, + 74.0, + 74.0 + ] + }, + { + "name": "storage.remote_bytes", + "display_name": "Filesystem remote after push", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 17267763.5, + "median_absolute_deviation": 0.5, + "samples": [ + 17267764.0, + 17267765.0, + 17267763.0, + 17267763.0 + ] + }, + { + "name": "storage.remote_segments_bytes", + "display_name": "Remote segments", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10592591.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10592591.0, + 10592591.0, + 10592591.0, + 10592591.0 + ] + }, + { + "name": "storage.remote_commits_bytes", + "display_name": "Remote storage commits", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 920.0, + "median_absolute_deviation": 0.0, + "samples": [ + 920.0, + 920.0, + 920.0, + 920.0 + ] + }, + { + "name": "storage.remote_objects_bytes", + "display_name": "Remote repository objects", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 382710.5, + "median_absolute_deviation": 0.5, + "samples": [ + 382711.0, + 382712.0, + 382710.0, + 382710.0 + ] + }, + { + "name": "storage.remote_payloads_bytes", + "display_name": "Remote external payloads", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 6291456.0, + "median_absolute_deviation": 0.0, + "samples": [ + 6291456.0, + 6291456.0, + 6291456.0, + 6291456.0 + ] + }, + { + "name": "storage.remote_metadata_bytes", + "display_name": "Remote refs and metadata", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 86.0, + "median_absolute_deviation": 0.0, + "samples": [ + 86.0, + 86.0, + 86.0, + 86.0 + ] + }, + { + "name": "storage.remote_file_count", + "display_name": "Remote file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 11.0, + "median_absolute_deviation": 0.0, + "samples": [ + 11.0, + 11.0, + 11.0, + 11.0 + ] + } + ] +} diff --git a/packages/graft-sdk/benchmark/results/git-workflow-comparison.md b/packages/graft-sdk/benchmark/results/git-workflow-comparison.md new file mode 100644 index 00000000..061149c4 --- /dev/null +++ b/packages/graft-sdk/benchmark/results/git-workflow-comparison.md @@ -0,0 +1,44 @@ + +## Graft performance + +Fixed `ci` dataset, 4 aligned base/candidate pairs after 1 warmup pair(s). Negative paired change is better for every metric. + +### Speed + +| Metric | Baseline | Candidate | Paired change | Paired MAD | +|---|---:|---:|---:|---:| +| Repository init | 4.22 ms | 4.19 ms | ⚪ -2.6% | 7.1% | +| Stage initial dataset | 955.88 ms | 977.75 ms | ⚪ +4.2% | 25.0% | +| Commit initial dataset | 520.48 ms | 482.13 ms | ⚪ -4.0% | 3.6% | +| Stage 10% row update | 539.73 ms | 525.38 ms | ⚪ -1.5% | 1.4% | +| Commit incremental update | 501.34 ms | 541.47 ms | 🔴 +5.5% | 2.5% | +| Row diff between commits | 557.77 ms | 613.24 ms | ⚪ +9.5% | 7.5% | +| Checkout parent revision | 589.10 ms | 544.21 ms | 🟢 -9.2% | 2.3% | +| Push to filesystem remote | 646.71 ms | 657.29 ms | ⚪ +2.5% | 3.5% | + +### Storage + +| Metric | Baseline | Candidate | Paired change | Paired MAD | +|---|---:|---:|---:|---:| +| Worktree dataset | 9.85 MiB | 9.85 MiB | ⚪ +0.0% | 0.0% | +| Materialized SQLite database | 5.60 MiB | 5.60 MiB | ⚪ +0.0% | 0.0% | +| .graft after initial commit | 9.66 MiB | 9.66 MiB | ⚪ +0.0% | 0.0% | +| .graft after incremental commit | 16.97 MiB | 16.97 MiB | ⚪ +0.0% | 0.0% | +| Incremental history growth | 7.31 MiB | 7.32 MiB | ⚪ +0.0% | 0.0% | +| Initial storage amplification | 0.981× | 0.981× | ⚪ +0.0% | 0.0% | +| Two-commit storage amplification | 1.723× | 1.723× | ⚪ +0.0% | 0.0% | +| SQLite snapshot store | 10.62 MiB | 10.62 MiB | ⚪ +0.0% | 0.0% | +| Repository objects | 366.34 KiB | 366.34 KiB | ⚪ +0.0% | 0.0% | +| External file payloads | 6.00 MiB | 6.00 MiB | ⚪ +0.0% | 0.0% | +| Refs, index, and metadata | 805 B | 805 B | ⚪ +0.0% | 0.0% | +| .graft file count | 101 | 103 | ⚪ +2.0% | 0.0% | +| Repository object file count | 74 | 74 | ⚪ +0.0% | 0.0% | +| Filesystem remote after push | 16.47 MiB | 16.47 MiB | ⚪ +0.0% | 0.0% | +| Remote segments | 10.10 MiB | 10.10 MiB | ⚪ +0.0% | 0.0% | +| Remote storage commits | 920 B | 920 B | ⚪ +0.0% | 0.0% | +| Remote repository objects | 373.74 KiB | 373.74 KiB | ⚪ +0.0% | 0.0% | +| Remote external payloads | 6.00 MiB | 6.00 MiB | ⚪ +0.0% | 0.0% | +| Remote refs and metadata | 86 B | 86 B | ⚪ +0.0% | 0.0% | +| Remote file count | 11 | 11 | ⚪ +0.0% | 0.0% | + +Baseline: `graft-sdk-v0.3.4` (`graft-tool 0.11.0`) · Candidate: `next` (`graft-tool 0.11.0`). Change is the median of aligned per-pair percentages; noise is their median absolute deviation. Storage uses apparent file bytes. diff --git a/packages/graft-sdk/benchmark/results/git-workflow-v0.3.4.json b/packages/graft-sdk/benchmark/results/git-workflow-v0.3.4.json new file mode 100644 index 00000000..839f9b72 --- /dev/null +++ b/packages/graft-sdk/benchmark/results/git-workflow-v0.3.4.json @@ -0,0 +1,454 @@ +{ + "schema_version": 2, + "label": "graft-sdk-v0.3.4", + "graft_version": "graft-tool 0.11.0", + "provenance": { + "run_id": "18c7e773140911c8-a324", + "report_kind": "paired_baseline", + "harness_label": "next", + "build_profile": "release", + "os": "macos", + "arch": "aarch64", + "runner_image": null, + "pair_order": [ + "baseline_first", + "candidate_first", + "baseline_first", + "candidate_first" + ] + }, + "parameters": { + "profile": "ci", + "sqlite_rows": 20000, + "updated_rows": 2000, + "row_payload_bytes": 256, + "text_file_count": 64, + "text_file_bytes": 4096, + "binary_file_count": 2, + "binary_file_bytes": 2097152 + }, + "sample_count": 4, + "warmup_count": 1, + "metrics": [ + { + "name": "speed.repo_init", + "display_name": "Repository init", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 4.2196665, + "median_absolute_deviation": 0.17339599999999988, + "samples": [ + 4.980917, + 3.992958, + 4.3397499999999996, + 4.099583 + ] + }, + { + "name": "speed.stage_initial", + "display_name": "Stage initial dataset", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 955.8813545, + "median_absolute_deviation": 25.724125500000014, + "samples": [ + 1548.056333, + 961.388209, + 950.3745, + 909.939958 + ] + }, + { + "name": "speed.commit_initial", + "display_name": "Commit initial dataset", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 520.4834585000001, + "median_absolute_deviation": 9.60002099999997, + "samples": [ + 481.03366600000004, + 526.975709, + 513.991208, + 533.19125 + ] + }, + { + "name": "speed.stage_incremental", + "display_name": "Stage 10% row update", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 539.72775, + "median_absolute_deviation": 11.94577099999998, + "samples": [ + 505.86633299999994, + 528.9625, + 550.493, + 552.8540419999999 + ] + }, + { + "name": "speed.commit_incremental", + "display_name": "Commit incremental update", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 501.3425625, + "median_absolute_deviation": 8.2267085, + "samples": [ + 504.571792, + 498.113333, + 488.118375, + 585.563458 + ] + }, + { + "name": "speed.row_diff", + "display_name": "Row diff between commits", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 557.7707085, + "median_absolute_deviation": 1.1172920000000204, + "samples": [ + 557.71375, + 555.593083, + 557.827667, + 562.090458 + ] + }, + { + "name": "speed.checkout_parent", + "display_name": "Checkout parent revision", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 589.104208, + "median_absolute_deviation": 26.546312999999998, + "samples": [ + 576.0822909999999, + 530.112875, + 602.126125, + 629.1749169999999 + ] + }, + { + "name": "speed.push_fs_remote", + "display_name": "Push to filesystem remote", + "group": "speed", + "unit": "milliseconds", + "lower_is_better": true, + "median": 646.710396, + "median_absolute_deviation": 7.0337504999999965, + "samples": [ + 634.075333, + 645.277958, + 662.6425, + 648.142834 + ] + }, + { + "name": "storage.worktree_bytes", + "display_name": "Worktree dataset", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10330112.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10330112.0, + 10330112.0, + 10330112.0, + 10330112.0 + ] + }, + { + "name": "storage.sqlite_bytes", + "display_name": "Materialized SQLite database", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 5873664.0, + "median_absolute_deviation": 0.0, + "samples": [ + 5873664.0, + 5873664.0, + 5873664.0, + 5873664.0 + ] + }, + { + "name": "storage.graft_initial_bytes", + "display_name": ".graft after initial commit", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10129082.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10129082.0, + 10129082.0, + 10129082.0, + 10129082.0 + ] + }, + { + "name": "storage.graft_incremental_bytes", + "display_name": ".graft after incremental commit", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 17799179.0, + "median_absolute_deviation": 0.0, + "samples": [ + 17799179.0, + 17799179.0, + 17799179.0, + 17799179.0 + ] + }, + { + "name": "storage.incremental_growth_bytes", + "display_name": "Incremental history growth", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 7670097.0, + "median_absolute_deviation": 0.0, + "samples": [ + 7670097.0, + 7670097.0, + 7670097.0, + 7670097.0 + ] + }, + { + "name": "storage.initial_amplification", + "display_name": "Initial storage amplification", + "group": "storage", + "unit": "ratio", + "lower_is_better": true, + "median": 0.9805394171912173, + "median_absolute_deviation": 0.0, + "samples": [ + 0.9805394171912173, + 0.9805394171912173, + 0.9805394171912173, + 0.9805394171912173 + ] + }, + { + "name": "storage.incremental_amplification", + "display_name": "Two-commit storage amplification", + "group": "storage", + "unit": "ratio", + "lower_is_better": true, + "median": 1.723038336854431, + "median_absolute_deviation": 0.0, + "samples": [ + 1.723038336854431, + 1.723038336854431, + 1.723038336854431, + 1.723038336854431 + ] + }, + { + "name": "storage.fjall_incremental_bytes", + "display_name": "SQLite snapshot store", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 11131789.0, + "median_absolute_deviation": 0.0, + "samples": [ + 11131789.0, + 11131789.0, + 11131789.0, + 11131789.0 + ] + }, + { + "name": "storage.objects_incremental_bytes", + "display_name": "Repository objects", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 375129.0, + "median_absolute_deviation": 0.0, + "samples": [ + 375129.0, + 375129.0, + 375129.0, + 375129.0 + ] + }, + { + "name": "storage.payloads_incremental_bytes", + "display_name": "External file payloads", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 6291456.0, + "median_absolute_deviation": 0.0, + "samples": [ + 6291456.0, + 6291456.0, + 6291456.0, + 6291456.0 + ] + }, + { + "name": "storage.metadata_incremental_bytes", + "display_name": "Refs, index, and metadata", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 805.0, + "median_absolute_deviation": 0.0, + "samples": [ + 805.0, + 805.0, + 805.0, + 805.0 + ] + }, + { + "name": "storage.graft_file_count", + "display_name": ".graft file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 101.0, + "median_absolute_deviation": 0.0, + "samples": [ + 101.0, + 101.0, + 101.0, + 101.0 + ] + }, + { + "name": "storage.objects_file_count", + "display_name": "Repository object file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 74.0, + "median_absolute_deviation": 0.0, + "samples": [ + 74.0, + 74.0, + 74.0, + 74.0 + ] + }, + { + "name": "storage.remote_bytes", + "display_name": "Filesystem remote after push", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 17267135.0, + "median_absolute_deviation": 0.0, + "samples": [ + 17267135.0, + 17267135.0, + 17267135.0, + 17267136.0 + ] + }, + { + "name": "storage.remote_segments_bytes", + "display_name": "Remote segments", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 10591962.0, + "median_absolute_deviation": 0.0, + "samples": [ + 10591962.0, + 10591962.0, + 10591962.0, + 10591962.0 + ] + }, + { + "name": "storage.remote_commits_bytes", + "display_name": "Remote storage commits", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 920.0, + "median_absolute_deviation": 0.0, + "samples": [ + 920.0, + 920.0, + 920.0, + 920.0 + ] + }, + { + "name": "storage.remote_objects_bytes", + "display_name": "Remote repository objects", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 382711.0, + "median_absolute_deviation": 0.0, + "samples": [ + 382711.0, + 382711.0, + 382711.0, + 382712.0 + ] + }, + { + "name": "storage.remote_payloads_bytes", + "display_name": "Remote external payloads", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 6291456.0, + "median_absolute_deviation": 0.0, + "samples": [ + 6291456.0, + 6291456.0, + 6291456.0, + 6291456.0 + ] + }, + { + "name": "storage.remote_metadata_bytes", + "display_name": "Remote refs and metadata", + "group": "storage", + "unit": "bytes", + "lower_is_better": true, + "median": 86.0, + "median_absolute_deviation": 0.0, + "samples": [ + 86.0, + 86.0, + 86.0, + 86.0 + ] + }, + { + "name": "storage.remote_file_count", + "display_name": "Remote file count", + "group": "storage", + "unit": "count", + "lower_is_better": true, + "median": 11.0, + "median_absolute_deviation": 0.0, + "samples": [ + 11.0, + 11.0, + 11.0, + 11.0 + ] + } + ] +} diff --git a/packages/graft-sdk/benchmark/results/performance-matrix-candidate-macos-arm64.json b/packages/graft-sdk/benchmark/results/performance-matrix-candidate-macos-arm64.json new file mode 100644 index 00000000..face3168 --- /dev/null +++ b/packages/graft-sdk/benchmark/results/performance-matrix-candidate-macos-arm64.json @@ -0,0 +1,1135 @@ +{ + "schema": "graft-sdk-performance-matrix-v1", + "generated_at": "2026-08-02T05:55:14.481Z", + "source_revision": "76a0219bf1c24f329f360d2a407cc453d2c73fec", + "sdk": { + "module": "workspace-sdk", + "version": "0.3.4" + }, + "profile": "full", + "methodology": { + "iterations": 5, + "timing": "wall-clock milliseconds via performance.now()", + "percentiles": "nearest-rank over repeated resident-session reads", + "fixture_generation": "reported separately and excluded from Graft operation timings", + "isolation": "one fresh repository and Node.js child process per scale point" + }, + "environment": { + "platform": "darwin", + "arch": "arm64", + "node": "v26.0.0", + "cpu": "Apple M2", + "cpu_count": 8, + "memory_bytes": 25769803776, + "os": "Darwin 24.6.0" + }, + "total_wall_milliseconds": 594433.236, + "files": [ + { + "fixture": { + "file_count": 100, + "file_bytes": 256, + "change_counts": [ + 1, + 10, + 100 + ] + }, + "fixture_generation": { + "milliseconds": 9.786, + "response_bytes": 0 + }, + "repository_bytes": 394721, + "peak_rss_bytes": 67829760, + "operations": { + "session_open": { + "milliseconds": 421.031, + "response_bytes": 2 + }, + "init": { + "milliseconds": 441.583, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 48.558, + "response_bytes": 9353 + }, + "commit_initial": { + "milliseconds": 3.446, + "response_bytes": 9650 + }, + "clean_status_hot": { + "samples": [ + 24.971, + 1.893, + 1.882, + 1.817, + 1.817 + ], + "min": 1.817, + "p50": 1.882, + "p95": 24.971, + "max": 24.971, + "response_bytes": 1203 + }, + "history_summaries_50": { + "samples": [ + 0.283, + 0.098, + 0.093, + 0.091, + 0.088 + ], + "min": 0.088, + "p50": 0.093, + "p95": 0.283, + "max": 0.283, + "response_bytes": 1380 + }, + "reopen": { + "milliseconds": 527.318, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 13.022, + "response_bytes": 1203 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 19.034, + "response_bytes": 1532 + }, + "explicit_path_diff": { + "milliseconds": 0.753, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 1.166, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 1.826, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 19.092, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 10, + "status_dirty": { + "milliseconds": 19.671, + "response_bytes": 4341 + }, + "explicit_path_diff": { + "milliseconds": 3.033, + "response_bytes": 8545 + }, + "stage_paths": { + "milliseconds": 11.055, + "response_bytes": 2831 + }, + "commit": { + "milliseconds": 1.964, + "response_bytes": 1378 + }, + "status_after_commit": { + "milliseconds": 20.049, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 21.186, + "response_bytes": 32422 + }, + "explicit_path_diff": { + "milliseconds": 31.077, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 175.669, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 2.88, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 25.184, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 2010.016 + }, + { + "fixture": { + "file_count": 1000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 71.085, + "response_bytes": 0 + }, + "repository_bytes": 3778374, + "peak_rss_bytes": 174063616, + "operations": { + "session_open": { + "milliseconds": 503.669, + "response_bytes": 2 + }, + "init": { + "milliseconds": 409.709, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 215.972, + "response_bytes": 93053 + }, + "commit_initial": { + "milliseconds": 12.265, + "response_bytes": 93350 + }, + "clean_status_hot": { + "samples": [ + 100.843, + 13.121, + 12.773, + 13.077, + 12.569 + ], + "min": 12.569, + "p50": 13.077, + "p95": 100.843, + "max": 100.843, + "response_bytes": 1206 + }, + "history_summaries_50": { + "samples": [ + 0.254, + 0.117, + 0.105, + 0.098, + 0.098 + ], + "min": 0.098, + "p50": 0.105, + "p95": 0.254, + "max": 0.254, + "response_bytes": 1385 + }, + "reopen": { + "milliseconds": 443.246, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 59.138, + "response_bytes": 1205 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 86.105, + "response_bytes": 1534 + }, + "explicit_path_diff": { + "milliseconds": 0.504, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 4.162, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 6.121, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 100.921, + "response_bytes": 1208 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 92.771, + "response_bytes": 32424 + }, + "explicit_path_diff": { + "milliseconds": 24.812, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 606.529, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 8.589, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 103.174, + "response_bytes": 1208 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 100.294, + "response_bytes": 313225 + }, + "explicit_path_diff": { + "milliseconds": 24.141, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 13680.932, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 17.689, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 140.299, + "response_bytes": 1208 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 17172.872 + }, + { + "fixture": { + "file_count": 10000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 600.09, + "response_bytes": 0 + }, + "repository_bytes": 30157379, + "peak_rss_bytes": 300498944, + "operations": { + "session_open": { + "milliseconds": 481.789, + "response_bytes": 2 + }, + "init": { + "milliseconds": 403.403, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 2165.45, + "response_bytes": 930053 + }, + "commit_initial": { + "milliseconds": 133.184, + "response_bytes": 930350 + }, + "clean_status_hot": { + "samples": [ + 1198.409, + 138.295, + 137.78, + 137.59, + 140.246 + ], + "min": 137.59, + "p50": 138.295, + "p95": 1198.409, + "max": 1198.409, + "response_bytes": 1209 + }, + "history_summaries_50": { + "samples": [ + 0.351, + 0.195, + 0.151, + 0.168, + 0.115 + ], + "min": 0.115, + "p50": 0.168, + "p95": 0.351, + "max": 0.351, + "response_bytes": 1386 + }, + "reopen": { + "milliseconds": 421.152, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 333.305, + "response_bytes": 1208 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 913.149, + "response_bytes": 1537 + }, + "explicit_path_diff": { + "milliseconds": 0.687, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 51.951, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 69.624, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 998.388, + "response_bytes": 1210 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 869.077, + "response_bytes": 32427 + }, + "explicit_path_diff": { + "milliseconds": 29.518, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 5838.468, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 79.844, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 1082.308, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 952.677, + "response_bytes": 313228 + }, + "explicit_path_diff": { + "milliseconds": 26.24, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 66885.543, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 94.136, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 1157.115, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 86694.466 + }, + { + "fixture": { + "file_count": 50000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 2883.192, + "response_bytes": 0 + }, + "repository_bytes": 147379419, + "peak_rss_bytes": 713195520, + "operations": { + "session_open": { + "milliseconds": 373.198, + "response_bytes": 2 + }, + "init": { + "milliseconds": 410.027, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 14821.72, + "response_bytes": 4650053 + }, + "commit_initial": { + "milliseconds": 688.356, + "response_bytes": 4650350 + }, + "clean_status_hot": { + "samples": [ + 6059.452, + 721.583, + 722.502, + 725.639, + 718.417 + ], + "min": 718.417, + "p50": 722.502, + "p95": 6059.452, + "max": 6059.452, + "response_bytes": 1209 + }, + "history_summaries_50": { + "samples": [ + 0.688, + 0.129, + 0.109, + 0.104, + 0.104 + ], + "min": 0.104, + "p50": 0.109, + "p95": 0.688, + "max": 0.688, + "response_bytes": 1386 + }, + "reopen": { + "milliseconds": 418.151, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 1598.785, + "response_bytes": 1209 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 4674.521, + "response_bytes": 1538 + }, + "explicit_path_diff": { + "milliseconds": 0.892, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 268.178, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 377.285, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 5452.068, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 4831.895, + "response_bytes": 32428 + }, + "explicit_path_diff": { + "milliseconds": 32.187, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 31776.444, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 435.33, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 5578.923, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 4811.175, + "response_bytes": 313229 + }, + "explicit_path_diff": { + "milliseconds": 26.299, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 327958.755, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 444.161, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 15568.802, + "response_bytes": 1212 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 439391.84 + } + ], + "sqlite": [ + { + "fixture": { + "rows": 10000, + "payload_bytes": 64, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 753664 + }, + "fixture_generation": { + "milliseconds": 10.679, + "response_bytes": 0 + }, + "repository_bytes": 1218219, + "peak_rss_bytes": 82804736, + "operations": { + "session_open": { + "milliseconds": 439.723, + "response_bytes": 2 + }, + "init": { + "milliseconds": 400.102, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 11.733, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 8.596, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 17.586, + 0.846, + 0.488, + 0.56, + 0.454 + ], + "min": 0.454, + "p50": 0.56, + "p95": 17.586, + "max": 17.586, + "response_bytes": 1200 + }, + "history_summaries_50": { + "samples": [ + 0.266, + 0.115, + 0.1, + 0.098, + 0.096 + ], + "min": 0.096, + "p50": 0.1, + "p95": 0.266, + "max": 0.266, + "response_bytes": 1652 + }, + "reopen": { + "milliseconds": 500.24, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 1.66, + "response_bytes": 1200 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 12.955, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 17.874, + "response_bytes": 1116 + }, + "working_rows_first_100": { + "milliseconds": 9.959, + "response_bytes": 1356 + }, + "stage": { + "milliseconds": 4.673, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 12.365, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 13.216, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 10.21, + "response_bytes": 1231 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 11.096, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 9.341, + "response_bytes": 1117 + }, + "working_rows_first_100": { + "milliseconds": 7.427, + "response_bytes": 21650 + }, + "stage": { + "milliseconds": 4.601, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 6.852, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 14.27, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 9.351, + "response_bytes": 1232 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 13.496, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 8.43, + "response_bytes": 1119 + }, + "working_rows_first_100": { + "milliseconds": 1.236, + "response_bytes": 21499 + }, + "stage": { + "milliseconds": 5.997, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 12.917, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 14.441, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 10.926, + "response_bytes": 1235 + } + } + ], + "scenario_wall_milliseconds": 1758.161 + }, + { + "fixture": { + "rows": 100000, + "payload_bytes": 256, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 27385856 + }, + "fixture_generation": { + "milliseconds": 136.286, + "response_bytes": 0 + }, + "repository_bytes": 31540237, + "peak_rss_bytes": 239583232, + "operations": { + "session_open": { + "milliseconds": 404.297, + "response_bytes": 2 + }, + "init": { + "milliseconds": 476.649, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 86.485, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 55.502, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 34.633, + 0.271, + 0.206, + 0.203, + 0.197 + ], + "min": 0.197, + "p50": 0.206, + "p95": 34.633, + "max": 34.633, + "response_bytes": 1200 + }, + "history_summaries_50": { + "samples": [ + 0.352, + 0.127, + 0.12, + 0.104, + 0.104 + ], + "min": 0.104, + "p50": 0.12, + "p95": 0.352, + "max": 0.352, + "response_bytes": 1653 + }, + "reopen": { + "milliseconds": 488.042, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 0.666, + "response_bytes": 1199 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 11.627, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 214.022, + "response_bytes": 1119 + }, + "working_rows_first_100": { + "milliseconds": 111.946, + "response_bytes": 1744 + }, + "stage": { + "milliseconds": 69.135, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 44.5, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 38.444, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 194.427, + "response_bytes": 1234 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 18.985, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 192.131, + "response_bytes": 1121 + }, + "working_rows_first_100": { + "milliseconds": 113.353, + "response_bytes": 60153 + }, + "stage": { + "milliseconds": 79.009, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 45.709, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 35.736, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 195.525, + "response_bytes": 1236 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 17.602, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 192.074, + "response_bytes": 1123 + }, + "working_rows_first_100": { + "milliseconds": 19.757, + "response_bytes": 59999 + }, + "stage": { + "milliseconds": 130.226, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 233.181, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 37.799, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 240.802, + "response_bytes": 1238 + } + } + ], + "scenario_wall_milliseconds": 4161.638 + }, + { + "fixture": { + "rows": 1000000, + "payload_bytes": 384, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 410636288 + }, + "fixture_generation": { + "milliseconds": 1622.74, + "response_bytes": 0 + }, + "repository_bytes": 463502481, + "peak_rss_bytes": 808189952, + "operations": { + "session_open": { + "milliseconds": 433.325, + "response_bytes": 2 + }, + "init": { + "milliseconds": 408.843, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 1103.629, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 2255.79, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 534.951, + 0.33, + 0.234, + 0.212, + 0.198 + ], + "min": 0.198, + "p50": 0.234, + "p95": 534.951, + "max": 534.951, + "response_bytes": 1200 + }, + "history_summaries_50": { + "samples": [ + 0.568, + 0.138, + 0.12, + 0.106, + 0.099 + ], + "min": 0.099, + "p50": 0.12, + "p95": 0.568, + "max": 0.568, + "response_bytes": 1654 + }, + "reopen": { + "milliseconds": 855.884, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 0.669, + "response_bytes": 1199 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 13.66, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 3649.99, + "response_bytes": 1122 + }, + "working_rows_first_100": { + "milliseconds": 1675.274, + "response_bytes": 2003 + }, + "stage": { + "milliseconds": 1145.536, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1230.769, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 546.053, + "response_bytes": 1204 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 3973.779, + "response_bytes": 1237 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 13.489, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 3022.43, + "response_bytes": 1124 + }, + "working_rows_first_100": { + "milliseconds": 1728.666, + "response_bytes": 85855 + }, + "stage": { + "milliseconds": 1093.235, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1278.913, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 637.76, + "response_bytes": 1204 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 3921.808, + "response_bytes": 1239 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 14.5, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 3284.025, + "response_bytes": 1126 + }, + "working_rows_first_100": { + "milliseconds": 440.102, + "response_bytes": 85701 + }, + "stage": { + "milliseconds": 1504.853, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1499.509, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 602.307, + "response_bytes": 1204 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 4085.254, + "response_bytes": 1241 + } + } + ], + "scenario_wall_milliseconds": 43182.919 + } + ] +} diff --git a/packages/graft-sdk/benchmark/results/performance-matrix-v0.3.4-macos-arm64.json b/packages/graft-sdk/benchmark/results/performance-matrix-v0.3.4-macos-arm64.json new file mode 100644 index 00000000..09d4d4fd --- /dev/null +++ b/packages/graft-sdk/benchmark/results/performance-matrix-v0.3.4-macos-arm64.json @@ -0,0 +1,1135 @@ +{ + "schema": "graft-sdk-performance-matrix-v1", + "generated_at": "2026-08-02T04:14:42.994Z", + "source_revision": "76a0219bf1c24f329f360d2a407cc453d2c73fec", + "sdk": { + "module": "npm:@eidos.space/graft@0.3.4", + "version": "0.3.4" + }, + "profile": "full", + "methodology": { + "iterations": 5, + "timing": "wall-clock milliseconds via performance.now()", + "percentiles": "nearest-rank over repeated resident-session reads", + "fixture_generation": "reported separately and excluded from Graft operation timings", + "isolation": "one fresh repository and Node.js child process per scale point" + }, + "environment": { + "platform": "darwin", + "arch": "arm64", + "node": "v26.0.0", + "cpu": "Apple M2", + "cpu_count": 8, + "memory_bytes": 25769803776, + "os": "Darwin 24.6.0" + }, + "total_wall_milliseconds": 663020.005, + "files": [ + { + "fixture": { + "file_count": 100, + "file_bytes": 256, + "change_counts": [ + 1, + 10, + 100 + ] + }, + "fixture_generation": { + "milliseconds": 9.897, + "response_bytes": 0 + }, + "repository_bytes": 394461, + "peak_rss_bytes": 69025792, + "operations": { + "session_open": { + "milliseconds": 346.534, + "response_bytes": 2 + }, + "init": { + "milliseconds": 377.294, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 37.644, + "response_bytes": 9353 + }, + "commit_initial": { + "milliseconds": 2.5, + "response_bytes": 9650 + }, + "clean_status_hot": { + "samples": [ + 20.228, + 1.796, + 1.626, + 3.151, + 2.763 + ], + "min": 1.626, + "p50": 2.763, + "p95": 20.228, + "max": 20.228, + "response_bytes": 1203 + }, + "history_summaries_50": { + "samples": [ + 0.226, + 0.101, + 0.104, + 0.098, + 0.1 + ], + "min": 0.098, + "p50": 0.101, + "p95": 0.226, + "max": 0.226, + "response_bytes": 1380 + }, + "reopen": { + "milliseconds": 411.168, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 11.016, + "response_bytes": 1203 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 18.631, + "response_bytes": 1532 + }, + "explicit_path_diff": { + "milliseconds": 0.652, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 0.926, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 1.541, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 20.908, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 10, + "status_dirty": { + "milliseconds": 20.522, + "response_bytes": 4341 + }, + "explicit_path_diff": { + "milliseconds": 5.69, + "response_bytes": 8545 + }, + "stage_paths": { + "milliseconds": 15.117, + "response_bytes": 2831 + }, + "commit": { + "milliseconds": 2.032, + "response_bytes": 1378 + }, + "status_after_commit": { + "milliseconds": 21.154, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 25.499, + "response_bytes": 32422 + }, + "explicit_path_diff": { + "milliseconds": 31.818, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 175.697, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 3.101, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 26.482, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 1781.066 + }, + { + "fixture": { + "file_count": 1000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 66.761, + "response_bytes": 0 + }, + "repository_bytes": 3778374, + "peak_rss_bytes": 164872192, + "operations": { + "session_open": { + "milliseconds": 398.509, + "response_bytes": 2 + }, + "init": { + "milliseconds": 424.844, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 212.029, + "response_bytes": 93053 + }, + "commit_initial": { + "milliseconds": 12.214, + "response_bytes": 93350 + }, + "clean_status_hot": { + "samples": [ + 106.673, + 13.911, + 12.631, + 12.667, + 13.661 + ], + "min": 12.631, + "p50": 13.661, + "p95": 106.673, + "max": 106.673, + "response_bytes": 1206 + }, + "history_summaries_50": { + "samples": [ + 0.267, + 0.132, + 0.111, + 0.102, + 0.099 + ], + "min": 0.099, + "p50": 0.111, + "p95": 0.267, + "max": 0.267, + "response_bytes": 1385 + }, + "reopen": { + "milliseconds": 421.872, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 48.297, + "response_bytes": 1205 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 84.561, + "response_bytes": 1534 + }, + "explicit_path_diff": { + "milliseconds": 0.501, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 4.036, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 6.22, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 92.783, + "response_bytes": 1207 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 92.656, + "response_bytes": 32424 + }, + "explicit_path_diff": { + "milliseconds": 25.469, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 606.084, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 8.642, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 143.659, + "response_bytes": 1208 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 104.399, + "response_bytes": 313226 + }, + "explicit_path_diff": { + "milliseconds": 27.285, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 13611.555, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 18.3, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 195.075, + "response_bytes": 1208 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 17049.251 + }, + { + "fixture": { + "file_count": 10000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 687.001, + "response_bytes": 0 + }, + "repository_bytes": 30153123, + "peak_rss_bytes": 292323328, + "operations": { + "session_open": { + "milliseconds": 451.122, + "response_bytes": 2 + }, + "init": { + "milliseconds": 445.243, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 1973.379, + "response_bytes": 930053 + }, + "commit_initial": { + "milliseconds": 128.302, + "response_bytes": 930350 + }, + "clean_status_hot": { + "samples": [ + 1182.66, + 140.95, + 140.888, + 140.389, + 139.785 + ], + "min": 139.785, + "p50": 140.888, + "p95": 1182.66, + "max": 1182.66, + "response_bytes": 1209 + }, + "history_summaries_50": { + "samples": [ + 0.475, + 0.137, + 0.117, + 0.099, + 0.098 + ], + "min": 0.098, + "p50": 0.117, + "p95": 0.475, + "max": 0.475, + "response_bytes": 1386 + }, + "reopen": { + "milliseconds": 409.457, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 326.197, + "response_bytes": 1208 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 862.708, + "response_bytes": 1537 + }, + "explicit_path_diff": { + "milliseconds": 0.55, + "response_bytes": 1116 + }, + "stage_paths": { + "milliseconds": 46.073, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 62.479, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 1060.919, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 873.459, + "response_bytes": 32427 + }, + "explicit_path_diff": { + "milliseconds": 24.298, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 5804.495, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 80.849, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 1003.125, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 892.697, + "response_bytes": 313228 + }, + "explicit_path_diff": { + "milliseconds": 26.834, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 66342.869, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 90.574, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 2511.107, + "response_bytes": 1211 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 87213.111 + }, + { + "fixture": { + "file_count": 50000, + "file_bytes": 256, + "change_counts": [ + 1, + 100, + 1000 + ] + }, + "fixture_generation": { + "milliseconds": 2751.372, + "response_bytes": 0 + }, + "repository_bytes": 147388475, + "peak_rss_bytes": 594198528, + "operations": { + "session_open": { + "milliseconds": 393.027, + "response_bytes": 2 + }, + "init": { + "milliseconds": 409.962, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 16867.416, + "response_bytes": 4650053 + }, + "commit_initial": { + "milliseconds": 698.795, + "response_bytes": 4650350 + }, + "clean_status_hot": { + "samples": [ + 11623.115, + 731.433, + 737.362, + 736.639, + 771.895 + ], + "min": 731.433, + "p50": 737.362, + "p95": 11623.115, + "max": 11623.115, + "response_bytes": 1209 + }, + "history_summaries_50": { + "samples": [ + 0.811, + 0.129, + 0.109, + 0.101, + 0.099 + ], + "min": 0.099, + "p50": 0.109, + "p95": 0.811, + "max": 0.811, + "response_bytes": 1386 + }, + "reopen": { + "milliseconds": 418.991, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 1607.913, + "response_bytes": 1209 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_paths": 1, + "status_dirty": { + "milliseconds": 9302.095, + "response_bytes": 1538 + }, + "explicit_path_diff": { + "milliseconds": 1.856, + "response_bytes": 1117 + }, + "stage_paths": { + "milliseconds": 286.38, + "response_bytes": 320 + }, + "commit": { + "milliseconds": 371.203, + "response_bytes": 513 + }, + "status_after_commit": { + "milliseconds": 11597.153, + "response_bytes": 1212 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 100, + "status_dirty": { + "milliseconds": 8890.65, + "response_bytes": 32428 + }, + "explicit_path_diff": { + "milliseconds": 31.86, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 32116.405, + "response_bytes": 27941 + }, + "commit": { + "milliseconds": 439.304, + "response_bytes": 10019 + }, + "status_after_commit": { + "milliseconds": 12596.406, + "response_bytes": 1212 + }, + "status_after_commit_cache_hit": false + }, + { + "changed_paths": 1000, + "status_dirty": { + "milliseconds": 10235.255, + "response_bytes": 313230 + }, + "explicit_path_diff": { + "milliseconds": 37.574, + "response_bytes": 82799 + }, + "stage_paths": { + "milliseconds": 334414.343, + "response_bytes": 279041 + }, + "commit": { + "milliseconds": 447.149, + "response_bytes": 96420 + }, + "status_after_commit": { + "milliseconds": 13365.243, + "response_bytes": 1212 + }, + "status_after_commit_cache_hit": false + } + ], + "scenario_wall_milliseconds": 478912.886 + } + ], + "sqlite": [ + { + "fixture": { + "rows": 10000, + "payload_bytes": 64, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 753664 + }, + "fixture_generation": { + "milliseconds": 9.72, + "response_bytes": 0 + }, + "repository_bytes": 1217884, + "peak_rss_bytes": 87605248, + "operations": { + "session_open": { + "milliseconds": 451.76, + "response_bytes": 2 + }, + "init": { + "milliseconds": 436.456, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 16.898, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 7.44, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 19.4, + 0.437, + 0.19, + 0.191, + 0.138 + ], + "min": 0.138, + "p50": 0.191, + "p95": 19.4, + "max": 19.4, + "response_bytes": 1200 + }, + "history_summaries_50": { + "samples": [ + 0.276, + 0.117, + 0.107, + 0.102, + 0.101 + ], + "min": 0.101, + "p50": 0.107, + "p95": 0.276, + "max": 0.276, + "response_bytes": 1652 + }, + "reopen": { + "milliseconds": 465.45, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 1.569, + "response_bytes": 1200 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 15.687, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 22.361, + "response_bytes": 1116 + }, + "working_rows_first_100": { + "milliseconds": 15.557, + "response_bytes": 1357 + }, + "stage": { + "milliseconds": 4.967, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 10.694, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 16.312, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 10.229, + "response_bytes": 1231 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 13.134, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 14.382, + "response_bytes": 1118 + }, + "working_rows_first_100": { + "milliseconds": 14.813, + "response_bytes": 21651 + }, + "stage": { + "milliseconds": 6.022, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 7.93, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 16.864, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 10.708, + "response_bytes": 1233 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 15.422, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 13.449, + "response_bytes": 1120 + }, + "working_rows_first_100": { + "milliseconds": 9.314, + "response_bytes": 21499 + }, + "stage": { + "milliseconds": 6.66, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 13.059, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 16.526, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 12.031, + "response_bytes": 1235 + } + } + ], + "scenario_wall_milliseconds": 1828.049 + }, + { + "fixture": { + "rows": 100000, + "payload_bytes": 256, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 27385856 + }, + "fixture_generation": { + "milliseconds": 211.576, + "response_bytes": 0 + }, + "repository_bytes": 31524687, + "peak_rss_bytes": 256753664, + "operations": { + "session_open": { + "milliseconds": 425.419, + "response_bytes": 2 + }, + "init": { + "milliseconds": 434.9, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 130.219, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 52.238, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 76.356, + 0.125, + 0.094, + 0.083, + 0.074 + ], + "min": 0.074, + "p50": 0.094, + "p95": 76.356, + "max": 76.356, + "response_bytes": 1199 + }, + "history_summaries_50": { + "samples": [ + 0.335, + 0.135, + 0.118, + 0.109, + 0.155 + ], + "min": 0.109, + "p50": 0.135, + "p95": 0.335, + "max": 0.335, + "response_bytes": 1654 + }, + "reopen": { + "milliseconds": 517.456, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 0.673, + "response_bytes": 1199 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 53.97, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 263.524, + "response_bytes": 1119 + }, + "working_rows_first_100": { + "milliseconds": 270.433, + "response_bytes": 1744 + }, + "stage": { + "milliseconds": 129.149, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 48.901, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 90.871, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 237.851, + "response_bytes": 1234 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 56.462, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 263.284, + "response_bytes": 1121 + }, + "working_rows_first_100": { + "milliseconds": 265.114, + "response_bytes": 60153 + }, + "stage": { + "milliseconds": 95.913, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 45.833, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 78.188, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 237.389, + "response_bytes": 1236 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 63.834, + "response_bytes": 1515 + }, + "working_summary": { + "milliseconds": 271.194, + "response_bytes": 1123 + }, + "working_rows_first_100": { + "milliseconds": 175.721, + "response_bytes": 60000 + }, + "stage": { + "milliseconds": 140.81, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 233.942, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 97.516, + "response_bytes": 1203 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 270.413, + "response_bytes": 1238 + } + } + ], + "scenario_wall_milliseconds": 5498.179 + }, + { + "fixture": { + "rows": 1000000, + "payload_bytes": 384, + "change_counts": [ + 1, + 100, + 10000 + ], + "database_bytes": 410636288 + }, + "fixture_generation": { + "milliseconds": 1586.405, + "response_bytes": 0 + }, + "repository_bytes": 463299565, + "peak_rss_bytes": 694435840, + "operations": { + "session_open": { + "milliseconds": 412.623, + "response_bytes": 2 + }, + "init": { + "milliseconds": 424.229, + "response_bytes": 310 + }, + "stage_initial": { + "milliseconds": 2055.428, + "response_bytes": 146 + }, + "commit_initial": { + "milliseconds": 1458.804, + "response_bytes": 446 + }, + "clean_status_hot": { + "samples": [ + 1377.802, + 0.205, + 0.161, + 0.13, + 0.121 + ], + "min": 0.121, + "p50": 0.161, + "p95": 1377.802, + "max": 1377.802, + "response_bytes": 1199 + }, + "history_summaries_50": { + "samples": [ + 0.538, + 0.163, + 0.144, + 0.138, + 0.178 + ], + "min": 0.138, + "p50": 0.163, + "p95": 0.538, + "max": 0.538, + "response_bytes": 1655 + }, + "reopen": { + "milliseconds": 851.091, + "response_bytes": 2 + }, + "status_after_reopen": { + "milliseconds": 0.621, + "response_bytes": 1199 + }, + "persistent_status_hit": true + }, + "mutation_rounds": [ + { + "changed_rows": 1, + "status_dirty": { + "milliseconds": 730.357, + "response_bytes": 1516 + }, + "working_summary": { + "milliseconds": 4887.594, + "response_bytes": 1122 + }, + "working_rows_first_100": { + "milliseconds": 4737.522, + "response_bytes": 2003 + }, + "stage": { + "milliseconds": 1918.435, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1268.085, + "response_bytes": 512 + }, + "status_after_commit": { + "milliseconds": 1233.975, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 5184.715, + "response_bytes": 1237 + } + }, + { + "changed_rows": 100, + "status_dirty": { + "milliseconds": 787.243, + "response_bytes": 1516 + }, + "working_summary": { + "milliseconds": 4899.539, + "response_bytes": 1124 + }, + "working_rows_first_100": { + "milliseconds": 5098.979, + "response_bytes": 85855 + }, + "stage": { + "milliseconds": 1926.494, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1508.661, + "response_bytes": 514 + }, + "status_after_commit": { + "milliseconds": 1375.024, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 5707.404, + "response_bytes": 1239 + } + }, + { + "changed_rows": 10000, + "status_dirty": { + "milliseconds": 806.103, + "response_bytes": 1516 + }, + "working_summary": { + "milliseconds": 5310.682, + "response_bytes": 1126 + }, + "working_rows_first_100": { + "milliseconds": 3659.324, + "response_bytes": 85702 + }, + "stage": { + "milliseconds": 2155.779, + "response_bytes": 305 + }, + "commit": { + "milliseconds": 1581.089, + "response_bytes": 516 + }, + "status_after_commit": { + "milliseconds": 1128.444, + "response_bytes": 1205 + }, + "status_after_commit_cache_hit": false, + "historical_summary": { + "milliseconds": 5998.791, + "response_bytes": 1241 + } + } + ], + "scenario_wall_milliseconds": 70643.553 + } + ] +} diff --git a/packages/graft-sdk/benchmark/results/real-eidos-candidate-macos-arm64.json b/packages/graft-sdk/benchmark/results/real-eidos-candidate-macos-arm64.json new file mode 100644 index 00000000..7f7a663b --- /dev/null +++ b/packages/graft-sdk/benchmark/results/real-eidos-candidate-macos-arm64.json @@ -0,0 +1,74 @@ +{ + "schema": "graft-sdk-real-eidos-benchmark-v1", + "generated_at": "2026-08-02T05:55:33.332Z", + "sdk": { + "module": "workspace-sdk", + "version": "0.3.4" + }, + "source": { + "path": "fixture:Untitled.eidos", + "bytes": 460689408 + }, + "environment": { + "platform": "darwin", + "arch": "arm64", + "node": "v26.0.0", + "cpu": "Apple M2", + "memory_bytes": 25769803776 + }, + "fixture_copy": { + "milliseconds": 254.254, + "response_bytes": 0 + }, + "peak_rss_bytes": 705413120, + "operations": { + "session_open": { + "milliseconds": 500.714, + "response_bytes": 2 + }, + "init": { + "milliseconds": 410.435, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 1800.638, + "response_bytes": 149 + }, + "commit_initial": { + "milliseconds": 1978.35, + "response_bytes": 452 + }, + "status_dirty": { + "milliseconds": 19.742, + "response_bytes": 1523 + }, + "working_summary": { + "milliseconds": 1527.764, + "response_bytes": 1138 + }, + "working_meta_rows": { + "milliseconds": 1.689, + "response_bytes": 1610 + }, + "stage_meta_change": { + "milliseconds": 1580.63, + "response_bytes": 311 + }, + "commit_meta_change": { + "milliseconds": 1.692, + "response_bytes": 517 + }, + "status_after_commit": { + "milliseconds": 770.988, + "response_bytes": 1202 + }, + "historical_summary": { + "milliseconds": 4919.003, + "response_bytes": 1260 + }, + "history_summaries_50": { + "milliseconds": 0.316, + "response_bytes": 1340 + } + } +} diff --git a/packages/graft-sdk/benchmark/results/real-eidos-v0.3.4-macos-arm64.json b/packages/graft-sdk/benchmark/results/real-eidos-v0.3.4-macos-arm64.json new file mode 100644 index 00000000..70ee27d3 --- /dev/null +++ b/packages/graft-sdk/benchmark/results/real-eidos-v0.3.4-macos-arm64.json @@ -0,0 +1,74 @@ +{ + "schema": "graft-sdk-real-eidos-benchmark-v1", + "generated_at": "2026-08-02T05:31:04.466Z", + "sdk": { + "module": "npm:@eidos.space/graft@0.3.4", + "version": "0.3.4" + }, + "source": { + "path": "fixture:Untitled.eidos", + "bytes": 460689408 + }, + "environment": { + "platform": "darwin", + "arch": "arm64", + "node": "v26.0.0", + "cpu": "Apple M2", + "memory_bytes": 25769803776 + }, + "fixture_copy": { + "milliseconds": 1294.364, + "response_bytes": 0 + }, + "peak_rss_bytes": 4170727424, + "operations": { + "session_open": { + "milliseconds": 463.845, + "response_bytes": 2 + }, + "init": { + "milliseconds": 445.34, + "response_bytes": 308 + }, + "stage_initial": { + "milliseconds": 6366.419, + "response_bytes": 149 + }, + "commit_initial": { + "milliseconds": 3661.209, + "response_bytes": 452 + }, + "status_dirty": { + "milliseconds": 5395.603, + "response_bytes": 1525 + }, + "working_summary": { + "milliseconds": 18836.532, + "response_bytes": 1154 + }, + "working_meta_rows": { + "milliseconds": 10063.069, + "response_bytes": 1614 + }, + "stage_meta_change": { + "milliseconds": 6052.94, + "response_bytes": 311 + }, + "commit_meta_change": { + "milliseconds": 21298.491, + "response_bytes": 517 + }, + "status_after_commit": { + "milliseconds": 7451.076, + "response_bytes": 1203 + }, + "historical_summary": { + "milliseconds": 12528.875, + "response_bytes": 1269 + }, + "history_summaries_50": { + "milliseconds": 0.389, + "response_bytes": 1340 + } + } +} diff --git a/packages/graft-sdk/package.json b/packages/graft-sdk/package.json index e9643002..b895e986 100644 --- a/packages/graft-sdk/package.json +++ b/packages/graft-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@eidos.space/graft", - "version": "0.3.4", + "version": "0.3.5", "description": "Long-lived in-process Graft repository sessions for Node.js and Electron.", "license": "MIT OR Apache-2.0", "repository": { @@ -39,6 +39,8 @@ "test": "node --test test/*.test.js", "bench": "node benchmark/cli-vs-sdk.mjs", "bench:large": "node benchmark/large-repository.mjs", + "bench:matrix": "node benchmark/performance-matrix.mjs", + "bench:real-eidos": "node benchmark/real-eidos.mjs", "bench:sqlite-diff": "node benchmark/sqlite-diff.mjs", "bench:push": "node benchmark/push.mjs", "release:create-packages": "napi create-npm-dirs --npm-dir npm", diff --git a/packages/graft-sdk/test/repository-session.test.js b/packages/graft-sdk/test/repository-session.test.js index c746e0e6..8b36212f 100644 --- a/packages/graft-sdk/test/repository-session.test.js +++ b/packages/graft-sdk/test/repository-session.test.js @@ -7,6 +7,8 @@ const os = require("node:os") const path = require("node:path") const test = require("node:test") +const packageMetadata = require("../package.json") + let DatabaseSync try { ;({ DatabaseSync } = require("node:sqlite")) @@ -24,7 +26,7 @@ const { } = require("..") test("exposes ABI-stable SDK metadata and materialization contract", () => { - assert.equal(sdkVersion(), "0.3.4") + assert.equal(sdkVersion(), packageMetadata.version) for (const operation of [ "restore", "restorePaths", diff --git a/scripts/remote-release.test.mjs b/scripts/remote-release.test.mjs index 097b5d83..d08fbca3 100644 --- a/scripts/remote-release.test.mjs +++ b/scripts/remote-release.test.mjs @@ -60,7 +60,9 @@ test("validates the checked-in Remote package release contract", async () => { ); metadataByName.set(metadata.name, metadata); } - validatePackageMetadata(metadataByName, "0.2.0"); + const version = metadataByName.get("@eidos.space/graft-remote")?.version; + assert.equal(typeof version, "string"); + validatePackageMetadata(metadataByName, version); }); test("rejects a dependency that could escape the release version", () => { From 1ed9f8ab41936ac553806336a6883e96919c2255 Mon Sep 17 00:00:00 2001 From: Mayne Date: Sun, 2 Aug 2026 15:02:54 +0800 Subject: [PATCH 4/4] fix(sqlite): normalize volatile header counters in page index --- CHANGELOG.md | 2 + .../src/pragma/sqlite_worktree.rs | 120 +++++++++++++++--- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aed4f5fc..99cdc1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ avoiding a second read of the live application database. - Status refresh preserves proven local classification across Remote projection changes and refreshes ahead/behind metadata independently. +- SQLite page indexes use the same content semantics as the Graft VFS across raw and online-backup + snapshots, ignoring only SQLite's volatile page-1 counters and invalidating older cache schemas. ### Performance diff --git a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs index 52deab8d..dd51a022 100644 --- a/crates/graft-sqlite/src/pragma/sqlite_worktree.rs +++ b/crates/graft-sqlite/src/pragma/sqlite_worktree.rs @@ -18,7 +18,7 @@ use tempfile::TempDir; use super::*; const PAGE_HASH_CACHE_MAGIC: &[u8; 16] = b"graft-page-index"; -const PAGE_HASH_CACHE_VERSION: u32 = 2; +const PAGE_HASH_CACHE_VERSION: u32 = 3; const PAGE_HASH_BYTES: usize = 32; const PAGE_HASH_CHUNK_PAGES: usize = 64; const PAGE_HASH_CHUNK_BYTES: usize = PAGE_HASH_CHUNK_PAGES * PAGESIZE.as_usize(); @@ -28,8 +28,11 @@ const PAGE_SCAN_BUFFER_BYTES: usize = 4 * 1024 * 1024; const MIN_PAGE_HASH_CACHE_FILE_BYTES: u64 = 16 * 1024 * 1024; const MAX_PAGE_HASH_CACHE_ENTRIES: usize = 4; const MAX_WORKTREE_DIFF_PROBES: usize = 16; -const WORKTREE_DIFF_PROBE_VERSION: u32 = 1; +const WORKTREE_DIFF_PROBE_VERSION: u32 = 2; const MAX_PERSISTED_DIFF_PROBE_BYTES: u64 = 64 * 1024; +const SQLITE_FILE_CHANGE_COUNTER_OFFSET: usize = 24; +const SQLITE_VERSION_VALID_FOR_OFFSET: usize = 92; +const SQLITE_VOLATILE_HEADER_FIELD_BYTES: usize = 4; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] struct WorktreeFileFingerprint { @@ -149,14 +152,14 @@ impl SqlitePageHashCache { fn path_for_state(&self, state: &CommitFileState) -> Result { let state_hash = sqlite_page_index_state_hash(state)?; - Ok(self.directory.join(format!("pages-v2-{state_hash}.bin"))) + Ok(self.directory.join(format!("pages-v3-{state_hash}.bin"))) } fn probe_path_for_state(&self, state: &CommitFileState) -> Result { let state_hash = sqlite_page_index_state_hash(state)?; Ok(self .directory - .join(format!("worktree-probe-v1-{state_hash}.json"))) + .join(format!("worktree-probe-v2-{state_hash}.json"))) } fn load_probe( @@ -220,7 +223,7 @@ impl SqlitePageHashCache { })?; let final_path = self.probe_path_for_state(state)?; let temp_path = self.directory.join(format!( - ".worktree-probe-v1-{}-{}.tmp", + ".worktree-probe-v2-{}-{}.tmp", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) @@ -320,7 +323,7 @@ impl SqlitePageHashCache { std::fs::create_dir_all(&self.directory)?; let final_path = self.path_for_state(state)?; let temp_path = self.directory.join(format!( - ".pages-v2-{}-{}.tmp", + ".pages-v3-{}-{}.tmp", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) @@ -412,11 +415,11 @@ fn prune_page_hash_cache(directory: &Path, keep: &Path) { if let Some(state_hash) = path .file_name() .and_then(|name| name.to_str()) - .and_then(|name| name.strip_prefix("pages-v2-")) + .and_then(|name| name.strip_prefix("pages-v3-")) .and_then(|name| name.strip_suffix(".bin")) { let _ = std::fs::remove_file( - directory.join(format!("worktree-probe-v1-{state_hash}.json")), + directory.join(format!("worktree-probe-v2-{state_hash}.json")), ); } let _ = std::fs::remove_file(path); @@ -428,6 +431,41 @@ fn page_hash_chunk_count(page_count: usize) -> usize { page_count.div_ceil(PAGE_HASH_CHUNK_PAGES) } +/// Hashes SQLite pages using the same content semantics as the Graft VFS. +/// +/// SQLite's online backup rewrites the file change counter and version-valid-for number on page +/// 1. They are cache invalidation hints rather than database content, and the Graft VFS already +/// ignores writes that only touch these fields. Excluding them here keeps page indexes portable +/// between raw rollback-journal snapshots and online-backup snapshots. +fn sqlite_page_chunk_hash(first_page: u32, bytes: &[u8]) -> blake3::Hash { + if first_page != 1 { + return blake3::hash(bytes); + } + + let change_counter_end = SQLITE_FILE_CHANGE_COUNTER_OFFSET + SQLITE_VOLATILE_HEADER_FIELD_BYTES; + let version_valid_for_end = + SQLITE_VERSION_VALID_FOR_OFFSET + SQLITE_VOLATILE_HEADER_FIELD_BYTES; + let mut hasher = blake3::Hasher::new(); + hasher.update(&bytes[..SQLITE_FILE_CHANGE_COUNTER_OFFSET]); + hasher.update(&bytes[change_counter_end..SQLITE_VERSION_VALID_FOR_OFFSET]); + hasher.update(&bytes[version_valid_for_end..]); + hasher.finalize() +} + +fn sqlite_page_bytes_equal(page_number: u32, left: &[u8], right: &[u8]) -> bool { + if page_number != 1 { + return left == right; + } + + let change_counter_end = SQLITE_FILE_CHANGE_COUNTER_OFFSET + SQLITE_VOLATILE_HEADER_FIELD_BYTES; + let version_valid_for_end = + SQLITE_VERSION_VALID_FOR_OFFSET + SQLITE_VOLATILE_HEADER_FIELD_BYTES; + left[..SQLITE_FILE_CHANGE_COUNTER_OFFSET] == right[..SQLITE_FILE_CHANGE_COUNTER_OFFSET] + && left[change_counter_end..SQLITE_VERSION_VALID_FOR_OFFSET] + == right[change_counter_end..SQLITE_VERSION_VALID_FOR_OFFSET] + && left[version_valid_for_end..] == right[version_valid_for_end..] +} + /// A stable page reader for a physical `SQLite` worktree file. /// /// This module is the data-plane boundary between repository operations and `SQLite`. Repository @@ -630,7 +668,11 @@ impl PhysicalSqliteReader { format!("invalid SQLite page index {page_number}: {error}").into(), ) })?; - if self.read_page(pageidx)? != expected.read_page(pageidx)? { + if !sqlite_page_bytes_equal( + page_number, + self.read_page(pageidx)?.as_ref(), + expected.read_page(pageidx)?.as_ref(), + ) { changed_pages.insert(page_number); } } @@ -686,11 +728,19 @@ impl PhysicalSqliteReader { })?; let physical_page = self.read_page(pageidx)?; if page_number <= self.page_count().to_u32() - && physical_page != staged.read_page(pageidx)? + && !sqlite_page_bytes_equal( + page_number, + physical_page.as_ref(), + staged.read_page(pageidx)?.as_ref(), + ) { return Ok(None); } - if physical_page != previous.read_page(pageidx)? { + if !sqlite_page_bytes_equal( + page_number, + physical_page.as_ref(), + previous.read_page(pageidx)?.as_ref(), + ) { changed_pages.insert(page_number); } } @@ -759,7 +809,11 @@ impl PhysicalSqliteReader { let pageidx = PageIdx::try_from(page_number).map_err(|err| { ErrCtx::PragmaErr(format!("invalid SQLite page index {page_number}: {err}").into()) })?; - if self.read_page(pageidx)? != stored.read_page(pageidx)? { + if !sqlite_page_bytes_equal( + page_number, + self.read_page(pageidx)?.as_ref(), + stored.read_page(pageidx)?.as_ref(), + ) { return Ok(false); } } @@ -836,7 +890,7 @@ impl PhysicalSqliteReader { let mut changed_pages = BTreeSet::new(); self.visit_page_chunks(|first_page, chunk_bytes| { let chunk_index = (first_page - 1) as usize / PAGE_HASH_CHUNK_PAGES; - let current_hash = blake3::hash(chunk_bytes); + let current_hash = sqlite_page_chunk_hash(first_page, chunk_bytes); if cached_hashes .get(chunk_index) .is_some_and(|expected_hash| expected_hash == current_hash.as_bytes()) @@ -854,7 +908,11 @@ impl PhysicalSqliteReader { ) })?; let unchanged = expected.page_count().contains(pageidx) - && expected.read_page(pageidx)?.as_ref() == page_bytes; + && sqlite_page_bytes_equal( + page_number, + expected.read_page(pageidx)?.as_ref(), + page_bytes, + ); if !unchanged { changed_pages.insert(page_number); } @@ -1355,7 +1413,7 @@ fn import_sqlite_reader_state( let mut current_hashes = Vec::with_capacity(page_hash_chunk_count(page_count)); physical.visit_page_chunks(|first_page, chunk_bytes| { - let current_hash = *blake3::hash(chunk_bytes).as_bytes(); + let current_hash = *sqlite_page_chunk_hash(first_page, chunk_bytes).as_bytes(); current_hashes.push(current_hash); let chunk_index = (first_page - 1) as usize / PAGE_HASH_CHUNK_PAGES; if cached_hashes @@ -1374,9 +1432,11 @@ fn import_sqlite_reader_state( ) })?; let unchanged = match &base_reader { - Some(reader) if reader.page_count().contains(pageidx) => { - reader.read_page(pageidx)?.as_ref() == page_bytes - } + Some(reader) if reader.page_count().contains(pageidx) => sqlite_page_bytes_equal( + page_number, + reader.read_page(pageidx)?.as_ref(), + page_bytes, + ), _ => false, }; if unchanged { @@ -1554,6 +1614,30 @@ mod tests { prepare_physical_sqlite_file_state_with_cache(runtime, path, base, Some(&cache)) } + #[test] + fn page_index_ignores_only_sqlite_volatile_header_counters() { + let original = vec![0_u8; PAGESIZE.as_usize()]; + let mut counters_changed = original.clone(); + counters_changed[SQLITE_FILE_CHANGE_COUNTER_OFFSET..SQLITE_FILE_CHANGE_COUNTER_OFFSET + 4] + .copy_from_slice(&17_u32.to_be_bytes()); + counters_changed[SQLITE_VERSION_VALID_FOR_OFFSET..SQLITE_VERSION_VALID_FOR_OFFSET + 4] + .copy_from_slice(&23_u32.to_be_bytes()); + + assert!(sqlite_page_bytes_equal(1, &original, &counters_changed)); + assert_eq!( + sqlite_page_chunk_hash(1, &original), + sqlite_page_chunk_hash(1, &counters_changed) + ); + + let mut content_changed = counters_changed; + content_changed[40] = 1; + assert!(!sqlite_page_bytes_equal(1, &original, &content_changed)); + assert_ne!( + sqlite_page_chunk_hash(1, &original), + sqlite_page_chunk_hash(1, &content_changed) + ); + } + #[test] fn unchanged_import_reuses_snapshot_and_changed_import_is_incremental() { let temp = tempfile::tempdir().unwrap();