diff --git a/Cargo.toml b/Cargo.toml index 05507f4..65c740c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,20 +71,24 @@ predicates = "3" assert_matches = "1" trybuild = "1" -# Lint bar: batpak's workspace lints verbatim (the composition bar), plus -# pedantic-as-warn which texo already carried. Exceptions are per-site -# #[expect(..)] with a reason, never table-level allows. +# Lint bar: correctness restrictions plus the substantive API, ownership, and +# documentation checks used by this crate. Texo carries no lint suppression +# attributes or table-level allowances; diagnostics must be resolved at their +# structural source. The module-name style lint is intentionally outside this +# policy because public paths such as `backup::BackupManifest` are the stable, +# domain-readable API rather than implementation debt. [lints.rust] missing_docs = "warn" deprecated = "deny" [lints.clippy] -pedantic = { level = "warn", priority = -1 } dbg_macro = "deny" todo = "deny" unimplemented = "deny" unwrap_used = "deny" panic = "deny" +too_many_lines = "deny" +too_many_arguments = "deny" needless_return = "deny" print_stdout = "deny" print_stderr = "deny" @@ -96,9 +100,16 @@ disallowed_methods = "deny" large_enum_variant = "warn" clone_on_ref_ptr = "warn" needless_pass_by_value = "warn" -module_name_repetitions = "allow" -must_use_candidate = "allow" +must_use_candidate = "warn" missing_errors_doc = "warn" +doc_markdown = "warn" +ref_option = "warn" +wildcard_imports = "warn" +empty_line_after_doc_comments = "warn" +ptr_arg = "warn" +needless_borrow = "warn" +float_cmp = "warn" +similar_names = "warn" [profile.dev] debug = 1 diff --git a/deny.toml b/deny.toml index a6ce22e..5b28d85 100644 --- a/deny.toml +++ b/deny.toml @@ -2,15 +2,8 @@ targets = ["x86_64-unknown-linux-gnu"] all-features = true -[advisories] -ignore = [ - # paste: unmaintained build-time proc-macro, transitive via fastembed/image - # (local-onnx feature only; not in the default OpenRouter build). Low risk. - "RUSTSEC-2024-0436", -] - [licenses] -allow = ["MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "Unicode-3.0", "ISC", "Zlib", "OpenSSL", "MPL-2.0", "CDLA-Permissive-2.0"] +allow = ["MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "Unicode-3.0", "ISC", "Zlib", "CDLA-Permissive-2.0"] confidence-threshold = 0.8 [bans] diff --git a/justfile b/justfile index 68da910..76eaf73 100644 --- a/justfile +++ b/justfile @@ -36,6 +36,29 @@ test-hygiene: echo "test-hygiene: unwrap() banned in src/" exit 1 fi + if rg -n '\b(panic|todo|unimplemented|dbg|println|eprintln|print|eprint)!\s*\(' src/ tests/ 2>/dev/null; then + echo "test-hygiene: panic/debug/print macros are banned in src/ and tests/" + exit 1 + fi + if rg -n -U '#[[:space:]]*!?[[:space:]]*\[[[:space:]]*(allow|expect)[[:space:]]*\(|#[[:space:]]*!?[[:space:]]*\[[[:space:]]*cfg_attr[[:space:]]*\([^]]*(allow|expect)[[:space:]]*\(' src/ tests/ 2>/dev/null; then + echo "test-hygiene: lint suppression attributes are banned in src/ and tests/" + exit 1 + fi + if rg -n '^\s*[A-Za-z0-9_-]+\s*=\s*(\{[^}]*level\s*=\s*)?"allow"' Cargo.toml 2>/dev/null; then + echo "test-hygiene: Cargo lint levels may not be set to allow" + exit 1 + fi + oversized=0 + while IFS= read -r file; do + lines=$(wc -l < "$file") + if (( lines > 800 )); then + echo "test-hygiene: production Rust file exceeds 800 lines: $file ($lines)" + oversized=1 + fi + done < <(rg --files src -g '*.rs') + if (( oversized != 0 )); then + exit 1 + fi deny: cargo deny check @@ -51,15 +74,21 @@ test-prop: demo: cargo run --bin texo -- init --workspace demo cargo run --bin texo -- ingest sample_sources - cargo run --bin texo -- agent-context --out public/agent-context.json + cargo run --bin texo -- agent-context --allow-unsettled --out public/agent-context.json cargo run --bin texo -- check-staleness sample_sources/stale_onboarding.md --json || true - cargo run --bin texo -- compile --out public + cargo run --bin texo -- compile --allow-unsettled --out public demo-fresh: #!/usr/bin/env bash set -euo pipefail rm -rf .texo - rm -f public/* + rm -f \ + public/agent-context.json \ + public/claims.json \ + public/conflicts.json \ + public/index.html \ + public/onboarding.generated.md \ + public/stale-context.json touch public/.gitkeep just demo @@ -90,18 +119,39 @@ demo-helios: [workspaces.helios.semantics] enabled = true # Recall-favoring candidate thresholds: the judge is the correctness gate, - # so a lower floor only costs judge calls (bounded by cluster sizes). + # so a lower floor only costs judge calls (bounded per page by the pair budget). cosine_threshold = 0.5 relate_prefilter = 0.5 TOML echo "==> ingest (LLM extraction via texo-extract; first run hits OpenRouter, then cached)" "$TEXO" ingest examples/helios/docs echo "==> relate (semantic supersession + conflict pass; cached + resumable)" - set +e - "$TEXO" relate - RELATE_STATUS=$? - set -e - if [[ "$RELATE_STATUS" -ne 0 && "$RELATE_STATUS" -ne 2 ]]; then exit "$RELATE_STATUS"; fi + RELATE_CURSOR=0 + RELATE_COMPLETE=false + for _ in $(seq 1 100); do + set +e + RELATE_OUTPUT=$("$TEXO" relate --json --pair-budget 100000 --pair-cursor "$RELATE_CURSOR") + RELATE_STATUS=$? + set -e + if [[ "$RELATE_STATUS" -ne 0 && "$RELATE_STATUS" -ne 2 ]]; then + exit "$RELATE_STATUS" + fi + RELATE_OUTCOME=$(jq -er '.outcome' <<<"$RELATE_OUTPUT") + if [[ "$RELATE_OUTCOME" == "complete" ]]; then + RELATE_COMPLETE=true + break + fi + NEXT_CURSOR=$(jq -er '.next_candidate_cursor' <<<"$RELATE_OUTPUT") + if [[ "$NEXT_CURSOR" == "$RELATE_CURSOR" ]]; then + echo "relate cursor made no progress: $RELATE_CURSOR" >&2 + exit 1 + fi + RELATE_CURSOR=$NEXT_CURSOR + done + if [[ "$RELATE_COMPLETE" != true ]]; then + echo "relate did not complete within 100 bounded pages" >&2 + exit 1 + fi echo "==> compile onboarding -> public/helios/onboarding.generated.md" "$TEXO" compile --out public/helios echo diff --git a/src/backup.rs b/src/backup.rs index 155912b..4faeba7 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -9,1162 +9,26 @@ //! `manifest_hash_hex` outside the backup and supply it to //! [`verify_with_expected_manifest_hash`] when authenticity matters. -use std::ffi::OsString; -use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; -use std::path::{Component, Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; - -use batpak::store::backup_envelope::{ - backup_manifest_body_hash, restore_proof_evidence_report, BackupManifestBody, BackupSegmentRef, - BACKUP_MANIFEST_BODY_SCHEMA_VERSION, -}; -use batpak::store::{ - snapshot_report_body_hash, ReadOnly, SnapshotEvidenceReport, Store, StoreConfig, - SNAPSHOT_EVIDENCE_REPORT_SCHEMA_VERSION, -}; -use serde::{Deserialize, Serialize}; - -use crate::config::WorkspaceConfig; -use crate::error::TexoError; - /// Backup manifest schema. pub const MANIFEST_SCHEMA: &str = "texo.backup.v2"; -const MANIFEST_FILE: &str = "backup.json"; -const CONFIG_FILE: &str = "config.toml"; -const STORE_DIR: &str = "store"; -const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024; -const MAX_CONFIG_BYTES: u64 = 1024 * 1024; -const MAX_STORE_FILE_BYTES: u64 = 2 * 1024 * 1024 * 1024; -const MAX_STORE_FILES: usize = 100_000; -static VERIFY_COPY_COUNTER: AtomicU64 = AtomicU64::new(0); -static RESTORE_COUNTER: AtomicU64 = AtomicU64::new(0); - -/// One exact file recorded in backup evidence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct FileRecord { - /// File name relative to `store/`. - pub name: String, - /// Exact byte length. - pub bytes: u64, - /// BLAKE3 of the exact bytes. - pub hash_hex: String, -} - -/// Durable backup manifest. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct BackupManifest { - /// Manifest schema. - pub schema: String, - /// Captured workspace. - pub workspace_id: String, - /// Store path from workspace config, for operator-led restore. - pub store_path: String, - /// Creation time supplied by the CLI. - pub created_at_ms: u64, - /// `BatPak` lifecycle evidence binding the snapshot. - pub snapshot: SnapshotEvidenceReport, - /// BatPak-native canonical identity for the authority-bearing segment set. - pub substrate_manifest: BackupManifestBody, - /// Canonical digest of `substrate_manifest`, pinned inside the product envelope. - pub substrate_manifest_hash_hex: String, - /// Exact journal snapshot file table. - pub store_files: Vec, - /// Exact config size. - pub config_bytes: u64, - /// Exact config digest. - pub config_hash_hex: String, -} - -/// Successful backup creation report. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct BackupCreateReport { - /// Report schema. - pub schema: &'static str, - /// Absolute immutable destination. - pub dest: String, - /// Captured workspace. - pub workspace_id: String, - /// Journal files captured. - pub store_file_count: usize, - /// Journal bytes captured. - pub store_bytes: u64, - /// Snapshot structural identity. - pub snapshot_id_hex: String, - /// BatPak-native canonical segment-manifest digest. - pub substrate_manifest_hash_hex: String, - /// Evidence manifest digest. - pub manifest_hash_hex: String, -} - -/// One stable verification finding. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct BackupFinding { - /// Stable finding class. - pub kind: &'static str, - /// Sanitized evidence. - pub detail: String, -} - -/// Offline backup verification report. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct BackupVerifyReport { - /// Report schema. - pub schema: &'static str, - /// Whether every evidence check passed. - pub verified: bool, - /// Absolute destination inspected. - pub dest: String, - /// Workspace from a valid manifest, otherwise empty. - pub workspace_id: String, - /// Valid recorded store files. - pub store_files_valid: usize, - /// Expected recorded store files. - pub store_files_expected: usize, - /// Digest of manifest bytes found on disk. - pub manifest_hash_hex: String, - /// Content findings; empty on success. - pub findings: Vec, -} - -/// Successful restore into a fresh workspace root. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct BackupRestoreReport { - /// Report schema. - pub schema: &'static str, - /// Fresh workspace root published atomically. - pub dest: String, - /// Restored workspace id. - pub workspace_id: String, - /// Journal files restored. - pub store_file_count: usize, - /// Journal bytes restored. - pub store_bytes: u64, - /// Verified source manifest digest. - pub manifest_hash_hex: String, - /// Whether the restored `BatPak` chain verified before publication. - pub chain_verified: bool, -} - -/// Create a fresh evidence-backed backup. -/// -/// The destination directory is allocated exclusively and the manifest is -/// published last. `BatPak` evidence binds the final `store/` path, so staging -/// the snapshot under a renamed parent would invalidate that evidence. A crash -/// before manifest publication therefore leaves an honestly incomplete, -/// unverifiable directory rather than a falsely valid backup. -/// -/// # Errors -/// Returns an error for an existing/overlapping destination, unsafe source -/// files, snapshot failure, or a backup that fails its own verification. -pub fn create( - root: &Path, - workspace: &WorkspaceConfig, - store: &Store, - dest: &Path, - created_at_ms: u64, -) -> Result { - let dest = absolute_path(dest)?; - reject_overlap(root, workspace, &dest)?; - let destination = BackupDestination::create(&dest)?; - let store_dest = destination.path().join(STORE_DIR); - let snapshot = store.snapshot_with_evidence(&store_dest)?; - let store_files = hash_store_files(&store_dest)?; - let substrate_manifest = build_substrate_manifest(&snapshot, &store_files)?; - let substrate_manifest_hash_hex = hex_bytes( - &backup_manifest_body_hash(&substrate_manifest).map_err(|error| { - backup_error(format!("substrate manifest encoding failed: {error}")) - })?, - ); - let (config_hash_hex, config_bytes) = copy_config(root, destination.path())?; - let manifest = BackupManifest { - schema: MANIFEST_SCHEMA.to_string(), - workspace_id: workspace.workspace_id.clone(), - store_path: workspace.store_path.clone(), - created_at_ms, - snapshot: snapshot.clone(), - substrate_manifest, - substrate_manifest_hash_hex: substrate_manifest_hash_hex.clone(), - store_files: store_files.clone(), - config_bytes, - config_hash_hex, - }; - let mut manifest_bytes = serde_json::to_vec_pretty(&manifest)?; - manifest_bytes.push(b'\n'); - write_new_synced(&destination.path().join(MANIFEST_FILE), &manifest_bytes)?; - sync_directory(destination.path())?; - - let verified = verify(destination.path())?; - if !verified.verified { - return Err(backup_error(format!( - "prepared backup failed self-verification: {}", - verified - .findings - .iter() - .map(|finding| finding.kind) - .collect::>() - .join(", ") - ))); - } - destination.complete()?; - Ok(BackupCreateReport { - schema: "texo.backup-create.v1", - dest: dest.display().to_string(), - workspace_id: workspace.workspace_id.clone(), - store_file_count: store_files.len(), - store_bytes: store_files.iter().map(|record| record.bytes).sum(), - snapshot_id_hex: hex_bytes(&snapshot.body.snapshot_id), - substrate_manifest_hash_hex, - manifest_hash_hex: blake3::hash(&manifest_bytes).to_hex().to_string(), - }) -} - -/// Verify a backup using only bytes beneath its destination. -/// -/// Content failures are findings, not function errors. Function errors are -/// reserved for environmental failures such as unreadable directory entries. -/// -/// # Errors -/// Returns an error only when the destination cannot be safely inspected. -pub fn verify(dest: &Path) -> Result { - verify_with_expected_manifest_hash(dest, None) -} - -/// Verify a backup and optionally compare its manifest to an out-of-band pin. -/// -/// # Errors -/// Returns an input error for a malformed expected digest, or an environment -/// error when the destination cannot be safely inspected. -pub fn verify_with_expected_manifest_hash( - dest: &Path, - expected_manifest_hash: Option<&str>, -) -> Result { - if let Some(expected) = expected_manifest_hash { - if expected.len() != 64 || !expected.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(TexoError::OpInput { - op: "texo backup verify".to_string(), - detail: "expected manifest hash must be exactly 64 hexadecimal characters" - .to_string(), - }); - } - } - let original = dest.to_path_buf(); - let metadata = match fs::symlink_metadata(dest) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(empty_report( - &original, - finding("backup_missing", "backup destination does not exist"), - )); - } - Err(error) => return Err(error.into()), - }; - if !metadata.file_type().is_dir() { - return Ok(empty_report( - &original, - finding( - "backup_root_invalid", - "backup root must be a regular directory", - ), - )); - } - let dest = absolute_path(dest)?; - let manifest_path = dest.join(MANIFEST_FILE); - let manifest_bytes = match read_regular_bounded(&manifest_path, MAX_MANIFEST_BYTES) { - Ok(bytes) => bytes, - Err(detail) => return Ok(empty_report(&dest, finding("manifest_invalid", detail))), - }; - let manifest_hash_hex = blake3::hash(&manifest_bytes).to_hex().to_string(); - let manifest: BackupManifest = match serde_json::from_slice(&manifest_bytes) { - Ok(manifest) => manifest, - Err(error) => { - return Ok(empty_report_with_hash( - &dest, - manifest_hash_hex, - finding("manifest_invalid", error.to_string()), - )); - } - }; - let mut findings = Vec::new(); - if expected_manifest_hash.is_some_and(|expected| expected != manifest_hash_hex) { - findings.push(finding( - "manifest_hash_mismatch", - format!("manifest hash is {manifest_hash_hex}; it does not match the out-of-band pin"), - )); - } - if manifest.schema != MANIFEST_SCHEMA { - findings.push(finding( - "manifest_schema_unsupported", - format!("unsupported schema `{}`", manifest.schema), - )); - } - if manifest.store_files.len() > MAX_STORE_FILES { - findings.push(finding( - "manifest_file_limit", - format!("manifest records more than {MAX_STORE_FILES} store files"), - )); - } - check_top_level(&dest, &mut findings)?; - check_snapshot_evidence(&dest, &manifest, &mut findings); - check_substrate_restore_proof(&dest, &manifest, &mut findings); - let store_files_valid = check_store_files(&dest, &manifest, &mut findings)?; - let config_valid = check_config(&dest, &manifest, &mut findings); - if config_valid { - check_config_binding(&dest, &manifest, &mut findings); - } - if store_files_valid == manifest.store_files.len() { - check_store_read_only(&dest, &manifest, &mut findings); - } - Ok(BackupVerifyReport { - schema: "texo.backup-verify.v1", - verified: findings.is_empty(), - dest: dest.display().to_string(), - workspace_id: manifest.workspace_id, - store_files_valid, - store_files_expected: manifest.store_files.len(), - manifest_hash_hex, - findings, - }) -} - -/// Restore a verified backup into a fresh workspace root. -/// -/// The destination must not exist. Restore verifies the source first, copies -/// only manifest-listed regular files into a private sibling staging root, -/// rewrites the selected workspace store path to a safe root-relative default, -/// verifies the copied `BatPak` chain, and atomically renames the staging root -/// into place. Derived caches and client integration are intentionally absent. -/// -/// # Errors -/// Returns an error for an invalid backup, overlapping/existing destination, -/// unsafe source bytes, copied-file mismatch, or failed restored chain. -pub fn restore( - backup: &Path, - dest: &Path, - expected_manifest_hash: Option<&str>, -) -> Result { - let verification = verify_with_expected_manifest_hash(backup, expected_manifest_hash)?; - if !verification.verified { - return Err(backup_error(format!( - "backup restore refused: {}", - verification - .findings - .iter() - .map(|finding| finding.kind) - .collect::>() - .join(", ") - ))); - } - let backup = absolute_path(backup)?; - let dest = absolute_path(dest)?; - if dest.starts_with(&backup) || backup.starts_with(&dest) { - return Err(backup_error( - "restore destination must not overlap the backup", - )); - } - let manifest_bytes = read_regular_bounded(&backup.join(MANIFEST_FILE), MAX_MANIFEST_BYTES) - .map_err(backup_error)?; - let manifest: BackupManifest = serde_json::from_slice(&manifest_bytes)?; - let source_config = crate::config::TexoRootConfig::load(&backup.join(CONFIG_FILE)) - .map_err(|error| backup_error(error.to_string()))?; - let mut workspace = source_config - .workspaces - .get(&manifest.workspace_id) - .cloned() - .ok_or_else(|| backup_error("backup config does not contain its workspace"))?; - let restore_store_path = crate::config::WorkspaceEntry::for_id(&manifest.workspace_id) - .primary() - .map_err(|error| backup_error(error.to_string()))? - .store_path; - workspace - .set_primary_store_path(restore_store_path.clone()) - .map_err(|error| backup_error(error.to_string()))?; - let mut workspaces = std::collections::BTreeMap::new(); - workspaces.insert(manifest.workspace_id.clone(), workspace.clone()); - let restored_config = crate::config::TexoRootConfig { - default_workspace: manifest.workspace_id.clone(), - workspaces, - gateway: source_config.gateway, - }; - let destination = RestoreDestination::create(&dest)?; - let texo_dir = destination.path().join(".texo"); - fs::create_dir(&texo_dir)?; - let config_bytes = toml::to_string_pretty(&restored_config) - .map_err(|error| backup_error(error.to_string()))?; - write_new_synced(&texo_dir.join(CONFIG_FILE), config_bytes.as_bytes())?; - let store_dest = destination.path().join(&restore_store_path); - fs::create_dir_all(&store_dest)?; - copy_verified_store(&backup.join(STORE_DIR), &store_dest, &manifest.store_files)?; - let store = Store::::open_read_only(StoreConfig::new(&store_dest))?; - let chain = store.verify_chain()?; - if !chain.is_intact() { - return Err(backup_error(format!( - "restored store chain verification failed: {chain:?}" - ))); - } - drop(store); - sync_directory(&store_dest)?; - sync_directory(&texo_dir)?; - destination.complete()?; - Ok(BackupRestoreReport { - schema: "texo.backup-restore.v1", - dest: dest.display().to_string(), - workspace_id: manifest.workspace_id, - store_file_count: manifest.store_files.len(), - store_bytes: manifest.store_files.iter().map(|record| record.bytes).sum(), - manifest_hash_hex: verification.manifest_hash_hex, - chain_verified: true, - }) -} - -fn copy_verified_store( - source: &Path, - dest: &Path, - records: &[FileRecord], -) -> Result<(), TexoError> { - for record in records { - if !safe_flat_name(&record.name) { - return Err(backup_error("unsafe store record during restore")); - } - let target = dest.join(&record.name); - fs::copy(source.join(&record.name), &target)?; - File::open(&target)?.sync_all()?; - let (hash, bytes) = - hash_regular_bounded(&target, MAX_STORE_FILE_BYTES).map_err(backup_error)?; - if hash != record.hash_hex || bytes != record.bytes { - return Err(backup_error(format!( - "restored copy of {} differs from verified source", - record.name - ))); - } - } - Ok(()) -} - -fn build_substrate_manifest( - snapshot: &SnapshotEvidenceReport, - records: &[FileRecord], -) -> Result { - let segments = segment_refs_from_records(records)?; - let expected_ids = snapshot - .body - .copied_segment_ids_sorted - .iter() - .copied() - .collect::>(); - let observed_ids = segments - .iter() - .map(|segment| segment.segment_id) - .collect::>(); - if expected_ids != observed_ids { - return Err(backup_error( - "snapshot segment evidence does not match copied segment files", - )); - } - Ok(BackupManifestBody { - schema_version: BACKUP_MANIFEST_BODY_SCHEMA_VERSION, - backup_id: snapshot.body.snapshot_id, - layout_revision: 1, - tooling_revision: 1, - segments, - }) -} - -fn segment_refs_from_records(records: &[FileRecord]) -> Result, TexoError> { - let mut segments = Vec::new(); - for record in records { - if Path::new(&record.name).extension() != Some(std::ffi::OsStr::new("fbat")) { - continue; - } - let stem = record - .name - .strip_suffix(".fbat") - .ok_or_else(|| backup_error("segment filename has no numeric stem"))?; - let segment_id = stem - .parse::() - .map_err(|_| backup_error(format!("invalid segment filename `{}`", record.name)))?; - if format!("{segment_id:06}.fbat") != record.name { - return Err(backup_error(format!( - "non-canonical segment filename `{}`", - record.name - ))); - } - let bytes_digest = digest_from_hex(&record.hash_hex) - .ok_or_else(|| backup_error(format!("invalid segment digest for `{}`", record.name)))?; - segments.push(BackupSegmentRef { - segment_id, - bytes_digest, - }); - } - segments.sort(); - Ok(segments) -} - -fn digest_from_hex(value: &str) -> Option<[u8; 32]> { - if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return None; - } - let mut digest = [0_u8; 32]; - for (index, slot) in digest.iter_mut().enumerate() { - let offset = index * 2; - *slot = u8::from_str_radix(&value[offset..offset + 2], 16).ok()?; - } - Some(digest) -} - -fn check_substrate_restore_proof( - dest: &Path, - manifest: &BackupManifest, - findings: &mut Vec, -) { - if manifest.substrate_manifest.schema_version != BACKUP_MANIFEST_BODY_SCHEMA_VERSION { - findings.push(finding( - "substrate_manifest_schema_unsupported", - format!( - "unsupported substrate manifest schema {}", - manifest.substrate_manifest.schema_version - ), - )); - } - if manifest.substrate_manifest.backup_id != manifest.snapshot.body.snapshot_id { - findings.push(finding( - "substrate_manifest_snapshot_mismatch", - "substrate backup identity does not match snapshot identity", - )); - } - let claimed_hash = digest_from_hex(&manifest.substrate_manifest_hash_hex); - match backup_manifest_body_hash(&manifest.substrate_manifest) { - Ok(computed) if claimed_hash == Some(computed) => {} - Ok(_) => findings.push(finding( - "substrate_manifest_hash_mismatch", - "substrate manifest body hash does not recompute", - )), - Err(error) => findings.push(finding( - "substrate_manifest_invalid", - format!("substrate manifest cannot be encoded: {error}"), - )), - } - - let mut observed = Vec::new(); - let store_dir = dest.join(STORE_DIR); - let entries = match fs::read_dir(&store_dir) { - Ok(entries) => entries, - Err(error) => { - findings.push(finding( - "substrate_restore_proof_invalid", - error.to_string(), - )); - return; - } - }; - for entry in entries { - let entry = match entry { - Ok(entry) => entry, - Err(error) => { - findings.push(finding( - "substrate_restore_proof_invalid", - error.to_string(), - )); - return; - } - }; - let name = entry.file_name().to_string_lossy().to_string(); - let path = Path::new(&name); - if path.extension() != Some(std::ffi::OsStr::new("fbat")) { - continue; - } - let Some(stem) = path.file_stem().and_then(std::ffi::OsStr::to_str) else { - continue; - }; - let segment_id = match stem.parse::() { - Ok(segment_id) if format!("{segment_id:06}.fbat") == name => segment_id, - _ => { - findings.push(finding( - "substrate_restore_proof_invalid", - format!("invalid segment filename `{name}`"), - )); - continue; - } - }; - match hash_regular_bounded(&entry.path(), MAX_STORE_FILE_BYTES) { - Ok((hash, _bytes)) => match digest_from_hex(&hash) { - Some(bytes_digest) => observed.push(BackupSegmentRef { - segment_id, - bytes_digest, - }), - None => findings.push(finding( - "substrate_restore_proof_invalid", - format!("invalid digest for `{name}`"), - )), - }, - Err(detail) => findings.push(finding("substrate_restore_proof_invalid", detail)), - } - } - match restore_proof_evidence_report(&manifest.substrate_manifest, &observed) { - Ok(report) if report.body.findings.is_empty() => {} - Ok(report) => findings.push(finding( - "substrate_restore_proof_failed", - format!("BatPak restore proof findings: {:?}", report.body.findings), - )), - Err(error) => findings.push(finding( - "substrate_restore_proof_invalid", - format!("BatPak restore proof cannot be encoded: {error}"), - )), - } -} - -fn check_snapshot_evidence( - dest: &Path, - manifest: &BackupManifest, - findings: &mut Vec, -) { - if manifest.snapshot.body.schema_version != SNAPSHOT_EVIDENCE_REPORT_SCHEMA_VERSION { - findings.push(finding( - "snapshot_schema_unsupported", - format!( - "unsupported snapshot schema {}", - manifest.snapshot.body.schema_version - ), - )); - } - match snapshot_report_body_hash(&manifest.snapshot.body) { - Ok(hash) if hash == manifest.snapshot.body_hash => {} - Ok(_) => findings.push(finding( - "snapshot_evidence_mismatch", - "snapshot evidence body hash does not recompute", - )), - Err(error) => findings.push(finding( - "snapshot_evidence_mismatch", - format!("snapshot evidence cannot be encoded: {error}"), - )), - } - let path_hash = blake3::hash(dest.join(STORE_DIR).as_os_str().as_encoded_bytes()); - if path_hash.as_bytes() != &manifest.snapshot.body.destination_path_digest { - findings.push(finding( - "snapshot_destination_mismatch", - "snapshot evidence is bound to a different destination path", - )); - } -} - -fn check_store_files( - dest: &Path, - manifest: &BackupManifest, - findings: &mut Vec, -) -> Result { - let store_dir = dest.join(STORE_DIR); - match fs::symlink_metadata(&store_dir) { - Ok(metadata) if metadata.file_type().is_dir() => {} - Ok(_) => { - findings.push(finding( - "store_directory_invalid", - "store must be a regular directory, not a symbolic link or file", - )); - return Ok(0); - } - Err(error) => { - findings.push(finding("store_directory_invalid", error.to_string())); - return Ok(0); - } - } - let mut expected = std::collections::BTreeSet::new(); - let mut valid = 0; - for record in &manifest.store_files { - if !safe_flat_name(&record.name) || !expected.insert(record.name.clone()) { - findings.push(finding( - "store_record_invalid", - format!("unsafe or duplicate store file `{}`", record.name), - )); - continue; - } - let path = store_dir.join(&record.name); - match hash_regular_bounded(&path, MAX_STORE_FILE_BYTES) { - Ok((hash, bytes)) if hash == record.hash_hex && bytes == record.bytes => valid += 1, - Ok((hash, bytes)) => findings.push(finding( - "store_file_mismatch", - format!( - "{} expected {} bytes/{} but found {bytes}/{hash}", - record.name, record.bytes, record.hash_hex - ), - )), - Err(detail) => findings.push(finding("store_file_invalid", detail)), - } - } - match fs::read_dir(&store_dir) { - Ok(entries) => { - for entry in entries { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if !expected.contains(&name) { - findings.push(finding( - "unexpected_store_file", - format!("unrecorded store entry `{name}`"), - )); - } - } - } - Err(error) => findings.push(finding("store_directory_invalid", error.to_string())), - } - Ok(valid) -} - -fn check_config(dest: &Path, manifest: &BackupManifest, findings: &mut Vec) -> bool { - match hash_regular_bounded(&dest.join(CONFIG_FILE), MAX_CONFIG_BYTES) { - Ok((hash, bytes)) if hash == manifest.config_hash_hex && bytes == manifest.config_bytes => { - true - } - Ok((hash, bytes)) => { - findings.push(finding( - "config_mismatch", - format!( - "expected {}/{} but found {bytes}/{hash}", - manifest.config_bytes, manifest.config_hash_hex - ), - )); - false - } - Err(detail) => { - findings.push(finding("config_invalid", detail)); - false - } - } -} - -fn check_config_binding(dest: &Path, manifest: &BackupManifest, findings: &mut Vec) { - let config = match crate::config::TexoRootConfig::load(&dest.join(CONFIG_FILE)) { - Ok(config) => config, - Err(error) => { - findings.push(finding("config_binding_invalid", error.to_string())); - return; - } - }; - match config.resolve(Some(&manifest.workspace_id)) { - Ok(workspace) if workspace.store_path == manifest.store_path => {} - Ok(workspace) => findings.push(finding( - "config_binding_mismatch", - format!( - "manifest store path `{}` differs from config `{}`", - manifest.store_path, workspace.store_path - ), - )), - Err(error) => findings.push(finding("config_binding_mismatch", error.to_string())), - } -} - -fn check_store_read_only( - dest: &Path, - manifest: &BackupManifest, - findings: &mut Vec, -) { - let copy = match VerificationCopy::create(&dest.join(STORE_DIR), &manifest.store_files) { - Ok(copy) => copy, - Err(error) => { - findings.push(finding( - "store_open_invalid", - format!("snapshot cannot be prepared for read-only verification: {error}"), - )); - return; - } - }; - match Store::::open_read_only(StoreConfig::new(copy.path())) { - Ok(store) => match store.verify_chain() { - Ok(chain) if chain.is_intact() => {} - Ok(chain) => findings.push(finding( - "store_chain_invalid", - format!("snapshot chain verification failed: {chain:?}"), - )), - Err(error) => findings.push(finding( - "store_chain_invalid", - format!("snapshot chain verification errored: {error}"), - )), - }, - Err(error) => findings.push(finding( - "store_open_invalid", - format!("snapshot cannot open read-only: {error}"), - )), - } -} - -struct VerificationCopy { - path: PathBuf, -} - -impl VerificationCopy { - fn create(source: &Path, records: &[FileRecord]) -> Result { - let parent = std::env::temp_dir(); - for _attempt in 0..100 { - let counter = VERIFY_COPY_COUNTER.fetch_add(1, Ordering::Relaxed); - let path = parent.join(format!( - "texo-backup-verify-{}-{counter}", - std::process::id() - )); - match fs::create_dir(&path) { - Ok(()) => { - let copy = Self { path }; - for record in records { - if !safe_flat_name(&record.name) { - return Err(backup_error("unsafe store record in verification copy")); - } - fs::copy(source.join(&record.name), copy.path.join(&record.name))?; - } - return Ok(copy); - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error.into()), - } - } - Err(backup_error("could not allocate verification directory")) - } - - fn path(&self) -> &Path { - &self.path - } -} - -impl Drop for VerificationCopy { - fn drop(&mut self) { - let _ignored = fs::remove_dir_all(&self.path); - } -} - -fn check_top_level(dest: &Path, findings: &mut Vec) -> Result<(), TexoError> { - let expected = std::collections::BTreeSet::from([MANIFEST_FILE, CONFIG_FILE, STORE_DIR]); - for entry in fs::read_dir(dest)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if !expected.contains(name.as_str()) { - findings.push(finding( - "unexpected_backup_entry", - format!("unrecorded top-level entry `{name}`"), - )); - } - } - Ok(()) -} - -fn hash_store_files(directory: &Path) -> Result, TexoError> { - let mut records = Vec::new(); - for entry in fs::read_dir(directory)? { - let entry = entry?; - let metadata = entry.metadata()?; - if !metadata.file_type().is_file() { - return Err(backup_error(format!( - "snapshot entry `{}` is not a regular file", - entry.path().display() - ))); - } - let name = entry - .file_name() - .into_string() - .map_err(|_| backup_error("snapshot file name is not UTF-8"))?; - if records.len() >= MAX_STORE_FILES { - return Err(backup_error(format!( - "snapshot exceeds the {MAX_STORE_FILES}-file limit" - ))); - } - let (hash_hex, bytes) = - hash_regular_bounded(&entry.path(), MAX_STORE_FILE_BYTES).map_err(backup_error)?; - records.push(FileRecord { - name, - bytes, - hash_hex, - }); - } - records.sort_by(|left, right| left.name.cmp(&right.name)); - Ok(records) -} - -fn copy_config(root: &Path, dest: &Path) -> Result<(String, u64), TexoError> { - let source = root.join(".texo/config.toml"); - let bytes = read_regular_bounded(&source, MAX_CONFIG_BYTES).map_err(backup_error)?; - write_new_synced(&dest.join(CONFIG_FILE), &bytes)?; - Ok(( - blake3::hash(&bytes).to_hex().to_string(), - bytes.len() as u64, - )) -} - -fn read_regular_bounded(path: &Path, limit: u64) -> Result, String> { - let metadata = - fs::symlink_metadata(path).map_err(|error| format!("{}: {error}", path.display()))?; - if !metadata.file_type().is_file() { - return Err(format!("{} is not a regular file", path.display())); - } - if metadata.len() > limit { - return Err(format!("{} exceeds the {limit}-byte limit", path.display())); - } - fs::read(path).map_err(|error| format!("{}: {error}", path.display())) -} - -fn hash_regular_bounded(path: &Path, limit: u64) -> Result<(String, u64), String> { - let metadata = - fs::symlink_metadata(path).map_err(|error| format!("{}: {error}", path.display()))?; - if !metadata.file_type().is_file() || metadata.len() > limit { - return Err(format!("{} is not a bounded regular file", path.display())); - } - let mut file = File::open(path).map_err(|error| format!("{}: {error}", path.display()))?; - let mut hasher = blake3::Hasher::new(); - let mut bytes = 0_u64; - let mut buffer = vec![0_u8; 64 * 1024]; - loop { - let count = file - .read(&mut buffer) - .map_err(|error| format!("{}: {error}", path.display()))?; - if count == 0 { - break; - } - bytes = bytes.saturating_add(count as u64); - if bytes > limit { - return Err(format!( - "{} grew beyond the {limit}-byte limit", - path.display() - )); - } - hasher.update(&buffer[..count]); - } - Ok((hasher.finalize().to_hex().to_string(), bytes)) -} - -fn write_new_synced(path: &Path, bytes: &[u8]) -> Result<(), TexoError> { - let mut file = OpenOptions::new().create_new(true).write(true).open(path)?; - file.write_all(bytes)?; - file.sync_all()?; - Ok(()) -} - -fn reject_overlap(root: &Path, workspace: &WorkspaceConfig, dest: &Path) -> Result<(), TexoError> { - let root = absolute_path(root)?; - let texo = absolute_path(&root.join(".texo"))?; - let store = absolute_path(&workspace.store_path_buf(&root))?; - for live in [&root, &texo, &store] { - if dest.starts_with(live) || live.starts_with(dest) { - return Err(backup_error( - "backup destination must not overlap the workspace root, .texo, or live store", - )); - } - } - Ok(()) -} - -fn safe_flat_name(name: &str) -> bool { - !name.is_empty() - && Path::new(name) - .components() - .all(|part| matches!(part, Component::Normal(_))) - && Path::new(name).file_name().and_then(|value| value.to_str()) == Some(name) -} - -fn absolute_path(path: &Path) -> Result { - let absolute = std::path::absolute(path)?; - let mut normalized = PathBuf::new(); - for component in absolute.components() { - match component { - Component::CurDir => {} - Component::ParentDir => { - if !normalized.pop() { - return Err(backup_error("path escapes the filesystem root")); - } - } - kept @ (Component::Prefix(_) | Component::RootDir | Component::Normal(_)) => { - normalized.push(kept); - } - } - } - let mut existing = normalized.clone(); - let mut missing = Vec::::new(); - loop { - match fs::symlink_metadata(&existing) { - Ok(_) => { - let mut resolved = fs::canonicalize(&existing)?; - for name in missing.iter().rev() { - resolved.push(name); - } - return Ok(resolved); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let name = existing - .file_name() - .ok_or_else(|| backup_error("path has no existing ancestor"))?; - missing.push(name.to_os_string()); - if !existing.pop() { - return Err(backup_error("path has no existing ancestor")); - } - } - Err(error) => return Err(error.into()), - } - } -} - -fn sync_directory(path: &Path) -> Result<(), TexoError> { - File::open(path)?.sync_all()?; - Ok(()) -} - -fn finding(kind: &'static str, detail: impl Into) -> BackupFinding { - BackupFinding { - kind, - detail: detail.into(), - } -} - -fn backup_error(detail: impl Into) -> TexoError { - TexoError::Backup { - detail: detail.into(), - } -} - -fn hex_bytes(bytes: &[u8]) -> String { - use std::fmt::Write as _; - - bytes.iter().fold( - String::with_capacity(bytes.len().saturating_mul(2)), - |mut encoded, byte| { - let _result = write!(encoded, "{byte:02x}"); - encoded - }, - ) -} - -fn empty_report(dest: &Path, finding: BackupFinding) -> BackupVerifyReport { - empty_report_with_hash(dest, String::new(), finding) -} - -fn empty_report_with_hash( - dest: &Path, - manifest_hash_hex: String, - finding: BackupFinding, -) -> BackupVerifyReport { - BackupVerifyReport { - schema: "texo.backup-verify.v1", - verified: false, - dest: dest.display().to_string(), - workspace_id: String::new(), - store_files_valid: 0, - store_files_expected: 0, - manifest_hash_hex, - findings: vec![finding], - } -} - -struct BackupDestination { - path: PathBuf, - complete: bool, -} - -impl BackupDestination { - fn create(dest: &Path) -> Result { - match fs::symlink_metadata(dest) { - Ok(_) => return Err(backup_error("backup destination already exists")), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let parent = dest - .parent() - .ok_or_else(|| backup_error("backup destination has no parent"))?; - fs::create_dir_all(parent)?; - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt as _; - let mut builder = fs::DirBuilder::new(); - builder.mode(0o700).create(dest)?; - } - #[cfg(not(unix))] - fs::create_dir(dest)?; - Ok(Self { - path: dest.to_path_buf(), - complete: false, - }) - } - - fn path(&self) -> &Path { - &self.path - } - - fn complete(mut self) -> Result<(), TexoError> { - if let Some(parent) = self.path().parent() { - sync_directory(parent)?; - } - self.complete = true; - Ok(()) - } -} - -impl Drop for BackupDestination { - fn drop(&mut self) { - if !self.complete { - let _ignored = fs::remove_dir_all(&self.path); - } - } -} - -struct RestoreDestination { - final_path: PathBuf, - stage_path: PathBuf, - complete: bool, -} - -impl RestoreDestination { - fn create(dest: &Path) -> Result { - match fs::symlink_metadata(dest) { - Ok(_) => return Err(backup_error("restore destination already exists")), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let parent = dest - .parent() - .ok_or_else(|| backup_error("restore destination has no parent"))?; - fs::create_dir_all(parent)?; - for _attempt in 0..100 { - let sequence = RESTORE_COUNTER.fetch_add(1, Ordering::Relaxed); - let stage_path = - parent.join(format!(".texo-restore-{}-{sequence}", std::process::id())); - let created = { - let mut builder = fs::DirBuilder::new(); - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt as _; - builder.mode(0o700); - } - builder.create(&stage_path) - }; - match created { - Ok(()) => { - return Ok(Self { - final_path: dest.to_path_buf(), - stage_path, - complete: false, - }); - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error.into()), - } - } - Err(backup_error( - "could not allocate a private restore staging root", - )) - } - - fn path(&self) -> &Path { - &self.stage_path - } - - fn complete(mut self) -> Result<(), TexoError> { - sync_directory(&self.stage_path)?; - fs::rename(&self.stage_path, &self.final_path)?; - if let Some(parent) = self.final_path.parent() { - sync_directory(parent)?; - } - self.complete = true; - Ok(()) - } -} - -impl Drop for RestoreDestination { - fn drop(&mut self) { - if !self.complete { - let _ignored = fs::remove_dir_all(&self.stage_path); - } - } -} +pub(super) const MANIFEST_FILE: &str = "backup.json"; +pub(super) const CONFIG_FILE: &str = "config.toml"; +pub(super) const STORE_DIR: &str = "store"; +pub(super) const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024; +pub(super) const MAX_CONFIG_BYTES: u64 = 1024 * 1024; +pub(super) const MAX_STORE_FILE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +pub(super) const MAX_STORE_FILES: usize = 100_000; + +mod create; +mod filesystem; +mod model; +mod restore; +mod verify; + +pub use create::create; +pub use model::{ + BackupCreateReport, BackupFinding, BackupManifest, BackupRestoreReport, BackupVerifyReport, + FileRecord, +}; +pub use restore::restore; +pub use verify::{verify, verify_with_expected_manifest_hash}; diff --git a/src/backup/create.rs b/src/backup/create.rs new file mode 100644 index 0000000..a8619f9 --- /dev/null +++ b/src/backup/create.rs @@ -0,0 +1,203 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use batpak::store::backup_envelope::{ + backup_manifest_body_hash, BackupManifestBody, BackupSegmentRef, + BACKUP_MANIFEST_BODY_SCHEMA_VERSION, +}; +use batpak::store::{SnapshotEvidenceReport, Store}; + +use crate::config::WorkspaceConfig; +use crate::error::TexoError; + +use super::filesystem::{ + absolute_path, backup_error, copy_config, digest_from_hex, hash_store_files, hex_bytes, + reject_overlap, sync_directory, write_new_synced, +}; +use super::verify::verify; +use super::{ + BackupCreateReport, BackupManifest, FileRecord, MANIFEST_FILE, MANIFEST_SCHEMA, STORE_DIR, +}; + +/// Create a fresh evidence-backed backup. +/// +/// The destination directory is allocated exclusively and the manifest is +/// published last. `BatPak` evidence binds the final `store/` path, so staging +/// the snapshot under a renamed parent would invalidate that evidence. A crash +/// before manifest publication therefore leaves an honestly incomplete, +/// unverifiable directory rather than a falsely valid backup. +/// +/// # Errors +/// Returns an error for an existing/overlapping destination, unsafe source +/// files, snapshot failure, or a backup that fails its own verification. +pub fn create( + root: &Path, + workspace: &WorkspaceConfig, + store: &Store, + dest: &Path, + created_at_ms: u64, +) -> Result { + let dest = absolute_path(dest)?; + reject_overlap(root, workspace, &dest)?; + let destination = BackupDestination::create(&dest)?; + let store_dest = destination.path().join(STORE_DIR); + let snapshot = store.snapshot_with_evidence(&store_dest)?; + let store_files = hash_store_files(&store_dest)?; + let substrate_manifest = build_substrate_manifest(&snapshot, &store_files)?; + let substrate_manifest_hash_hex = hex_bytes( + &backup_manifest_body_hash(&substrate_manifest).map_err(|error| { + backup_error(format!("substrate manifest encoding failed: {error}")) + })?, + ); + let (config_hash_hex, config_bytes) = copy_config(root, destination.path())?; + let manifest = BackupManifest { + schema: MANIFEST_SCHEMA.to_string(), + workspace_id: workspace.workspace_id.clone(), + store_path: workspace.store_path.clone(), + created_at_ms, + snapshot: snapshot.clone(), + substrate_manifest, + substrate_manifest_hash_hex: substrate_manifest_hash_hex.clone(), + store_files: store_files.clone(), + config_bytes, + config_hash_hex, + }; + let mut manifest_bytes = serde_json::to_vec_pretty(&manifest)?; + manifest_bytes.push(b'\n'); + write_new_synced(&destination.path().join(MANIFEST_FILE), &manifest_bytes)?; + sync_directory(destination.path())?; + + let verified = verify(destination.path())?; + if !verified.verified { + return Err(backup_error(format!( + "prepared backup failed self-verification: {}", + verified + .findings + .iter() + .map(|finding| finding.kind) + .collect::>() + .join(", ") + ))); + } + destination.complete()?; + Ok(BackupCreateReport { + schema: "texo.backup-create.v1", + dest: dest.display().to_string(), + workspace_id: workspace.workspace_id.clone(), + store_file_count: store_files.len(), + store_bytes: store_files.iter().map(|record| record.bytes).sum(), + snapshot_id_hex: hex_bytes(&snapshot.body.snapshot_id), + substrate_manifest_hash_hex, + manifest_hash_hex: blake3::hash(&manifest_bytes).to_hex().to_string(), + }) +} + +fn build_substrate_manifest( + snapshot: &SnapshotEvidenceReport, + records: &[FileRecord], +) -> Result { + let segments = segment_refs_from_records(records)?; + let expected_ids = snapshot + .body + .copied_segment_ids_sorted + .iter() + .copied() + .collect::>(); + let observed_ids = segments + .iter() + .map(|segment| segment.segment_id) + .collect::>(); + if expected_ids != observed_ids { + return Err(backup_error( + "snapshot segment evidence does not match copied segment files", + )); + } + Ok(BackupManifestBody { + schema_version: BACKUP_MANIFEST_BODY_SCHEMA_VERSION, + backup_id: snapshot.body.snapshot_id, + layout_revision: 1, + tooling_revision: 1, + segments, + }) +} + +fn segment_refs_from_records(records: &[FileRecord]) -> Result, TexoError> { + let mut segments = Vec::new(); + for record in records { + if Path::new(&record.name).extension() != Some(std::ffi::OsStr::new("fbat")) { + continue; + } + let stem = record + .name + .strip_suffix(".fbat") + .ok_or_else(|| backup_error("segment filename has no numeric stem"))?; + let segment_id = stem + .parse::() + .map_err(|_| backup_error(format!("invalid segment filename `{}`", record.name)))?; + if format!("{segment_id:06}.fbat") != record.name { + return Err(backup_error(format!( + "non-canonical segment filename `{}`", + record.name + ))); + } + let bytes_digest = digest_from_hex(&record.hash_hex) + .ok_or_else(|| backup_error(format!("invalid segment digest for `{}`", record.name)))?; + segments.push(BackupSegmentRef { + segment_id, + bytes_digest, + }); + } + segments.sort(); + Ok(segments) +} + +struct BackupDestination { + path: PathBuf, + complete: bool, +} + +impl BackupDestination { + fn create(dest: &Path) -> Result { + match fs::symlink_metadata(dest) { + Ok(_) => return Err(backup_error("backup destination already exists")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let parent = dest + .parent() + .ok_or_else(|| backup_error("backup destination has no parent"))?; + fs::create_dir_all(parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt as _; + let mut builder = fs::DirBuilder::new(); + builder.mode(0o700).create(dest)?; + } + #[cfg(not(unix))] + fs::create_dir(dest)?; + Ok(Self { + path: dest.to_path_buf(), + complete: false, + }) + } + + fn path(&self) -> &Path { + &self.path + } + + fn complete(mut self) -> Result<(), TexoError> { + if let Some(parent) = self.path().parent() { + sync_directory(parent)?; + } + self.complete = true; + Ok(()) + } +} + +impl Drop for BackupDestination { + fn drop(&mut self) { + if !self.complete { + let _ignored = fs::remove_dir_all(&self.path); + } + } +} diff --git a/src/backup/filesystem.rs b/src/backup/filesystem.rs new file mode 100644 index 0000000..ab6d2e6 --- /dev/null +++ b/src/backup/filesystem.rs @@ -0,0 +1,210 @@ +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; + +use crate::config::WorkspaceConfig; +use crate::error::TexoError; + +use super::{ + BackupFinding, FileRecord, CONFIG_FILE, MAX_CONFIG_BYTES, MAX_STORE_FILES, MAX_STORE_FILE_BYTES, +}; + +pub(super) fn digest_from_hex(value: &str) -> Option<[u8; 32]> { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + let mut digest = [0_u8; 32]; + for (index, slot) in digest.iter_mut().enumerate() { + let offset = index * 2; + *slot = u8::from_str_radix(&value[offset..offset + 2], 16).ok()?; + } + Some(digest) +} + +pub(super) fn hash_store_files(directory: &Path) -> Result, TexoError> { + let mut records = Vec::new(); + for entry in fs::read_dir(directory)? { + let entry = entry?; + let metadata = entry.metadata()?; + if !metadata.file_type().is_file() { + return Err(backup_error(format!( + "snapshot entry `{}` is not a regular file", + entry.path().display() + ))); + } + let name = entry + .file_name() + .into_string() + .map_err(|_| backup_error("snapshot file name is not UTF-8"))?; + if records.len() >= MAX_STORE_FILES { + return Err(backup_error(format!( + "snapshot exceeds the {MAX_STORE_FILES}-file limit" + ))); + } + let (hash_hex, bytes) = + hash_regular_bounded(&entry.path(), MAX_STORE_FILE_BYTES).map_err(backup_error)?; + records.push(FileRecord { + name, + bytes, + hash_hex, + }); + } + records.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(records) +} + +pub(super) fn copy_config(root: &Path, dest: &Path) -> Result<(String, u64), TexoError> { + let source = root.join(".texo/config.toml"); + let bytes = read_regular_bounded(&source, MAX_CONFIG_BYTES).map_err(backup_error)?; + write_new_synced(&dest.join(CONFIG_FILE), &bytes)?; + Ok(( + blake3::hash(&bytes).to_hex().to_string(), + bytes.len() as u64, + )) +} + +pub(super) fn read_regular_bounded(path: &Path, limit: u64) -> Result, String> { + let metadata = + fs::symlink_metadata(path).map_err(|error| format!("{}: {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + if metadata.len() > limit { + return Err(format!("{} exceeds the {limit}-byte limit", path.display())); + } + fs::read(path).map_err(|error| format!("{}: {error}", path.display())) +} + +pub(super) fn hash_regular_bounded(path: &Path, limit: u64) -> Result<(String, u64), String> { + let metadata = + fs::symlink_metadata(path).map_err(|error| format!("{}: {error}", path.display()))?; + if !metadata.file_type().is_file() || metadata.len() > limit { + return Err(format!("{} is not a bounded regular file", path.display())); + } + let mut file = File::open(path).map_err(|error| format!("{}: {error}", path.display()))?; + let mut hasher = blake3::Hasher::new(); + let mut bytes = 0_u64; + let mut buffer = vec![0_u8; 64 * 1024]; + loop { + let count = file + .read(&mut buffer) + .map_err(|error| format!("{}: {error}", path.display()))?; + if count == 0 { + break; + } + bytes = bytes.saturating_add(count as u64); + if bytes > limit { + return Err(format!( + "{} grew beyond the {limit}-byte limit", + path.display() + )); + } + hasher.update(&buffer[..count]); + } + Ok((hasher.finalize().to_hex().to_string(), bytes)) +} + +pub(super) fn write_new_synced(path: &Path, bytes: &[u8]) -> Result<(), TexoError> { + let mut file = OpenOptions::new().create_new(true).write(true).open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + Ok(()) +} + +pub(super) fn reject_overlap( + root: &Path, + workspace: &WorkspaceConfig, + dest: &Path, +) -> Result<(), TexoError> { + let root = absolute_path(root)?; + let texo = absolute_path(&root.join(".texo"))?; + let store = absolute_path(&workspace.store_path_buf(&root))?; + for live in [&root, &texo, &store] { + if dest.starts_with(live) || live.starts_with(dest) { + return Err(backup_error( + "backup destination must not overlap the workspace root, .texo, or live store", + )); + } + } + Ok(()) +} + +pub(super) fn safe_flat_name(name: &str) -> bool { + !name.is_empty() + && Path::new(name) + .components() + .all(|part| matches!(part, Component::Normal(_))) + && Path::new(name).file_name().and_then(|value| value.to_str()) == Some(name) +} + +pub(super) fn absolute_path(path: &Path) -> Result { + let absolute = std::path::absolute(path)?; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + return Err(backup_error("path escapes the filesystem root")); + } + } + kept @ (Component::Prefix(_) | Component::RootDir | Component::Normal(_)) => { + normalized.push(kept); + } + } + } + let mut existing = normalized.clone(); + let mut missing = Vec::::new(); + loop { + match fs::symlink_metadata(&existing) { + Ok(_) => { + let mut resolved = fs::canonicalize(&existing)?; + for name in missing.iter().rev() { + resolved.push(name); + } + return Ok(resolved); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let name = existing + .file_name() + .ok_or_else(|| backup_error("path has no existing ancestor"))?; + missing.push(name.to_os_string()); + if !existing.pop() { + return Err(backup_error("path has no existing ancestor")); + } + } + Err(error) => return Err(error.into()), + } + } +} + +pub(super) fn sync_directory(path: &Path) -> Result<(), TexoError> { + File::open(path)?.sync_all()?; + Ok(()) +} + +pub(super) fn finding(kind: &'static str, detail: impl Into) -> BackupFinding { + BackupFinding { + kind, + detail: detail.into(), + } +} + +pub(super) fn backup_error(detail: impl Into) -> TexoError { + TexoError::Backup { + detail: detail.into(), + } +} + +pub(super) fn hex_bytes(bytes: &[u8]) -> String { + use std::fmt::Write as _; + + bytes.iter().fold( + String::with_capacity(bytes.len().saturating_mul(2)), + |mut encoded, byte| { + let _result = write!(encoded, "{byte:02x}"); + encoded + }, + ) +} diff --git a/src/backup/model.rs b/src/backup/model.rs new file mode 100644 index 0000000..52d187e --- /dev/null +++ b/src/backup/model.rs @@ -0,0 +1,113 @@ +//! Durable backup evidence and report shapes. + +use batpak::store::backup_envelope::BackupManifestBody; +use batpak::store::SnapshotEvidenceReport; +use serde::{Deserialize, Serialize}; + +/// One exact file recorded in backup evidence. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FileRecord { + /// File name relative to `store/`. + pub name: String, + /// Exact byte length. + pub bytes: u64, + /// BLAKE3 of the exact bytes. + pub hash_hex: String, +} + +/// Durable backup manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackupManifest { + /// Manifest schema. + pub schema: String, + /// Captured workspace. + pub workspace_id: String, + /// Store path from workspace config, for operator-led restore. + pub store_path: String, + /// Creation time supplied by the CLI. + pub created_at_ms: u64, + /// `BatPak` lifecycle evidence binding the snapshot. + pub snapshot: SnapshotEvidenceReport, + /// BatPak-native canonical identity for the authority-bearing segment set. + pub substrate_manifest: BackupManifestBody, + /// Canonical digest of `substrate_manifest`, pinned inside the product envelope. + pub substrate_manifest_hash_hex: String, + /// Exact journal snapshot file table. + pub store_files: Vec, + /// Exact config size. + pub config_bytes: u64, + /// Exact config digest. + pub config_hash_hex: String, +} + +/// Successful backup creation report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackupCreateReport { + /// Report schema. + pub schema: &'static str, + /// Absolute immutable destination. + pub dest: String, + /// Captured workspace. + pub workspace_id: String, + /// Journal files captured. + pub store_file_count: usize, + /// Journal bytes captured. + pub store_bytes: u64, + /// Snapshot structural identity. + pub snapshot_id_hex: String, + /// BatPak-native canonical segment-manifest digest. + pub substrate_manifest_hash_hex: String, + /// Evidence manifest digest. + pub manifest_hash_hex: String, +} + +/// One stable verification finding. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackupFinding { + /// Stable finding class. + pub kind: &'static str, + /// Sanitized evidence. + pub detail: String, +} + +/// Offline backup verification report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackupVerifyReport { + /// Report schema. + pub schema: &'static str, + /// Whether every evidence check passed. + pub verified: bool, + /// Absolute destination inspected. + pub dest: String, + /// Workspace from a valid manifest, otherwise empty. + pub workspace_id: String, + /// Valid recorded store files. + pub store_files_valid: usize, + /// Expected recorded store files. + pub store_files_expected: usize, + /// Digest of manifest bytes found on disk. + pub manifest_hash_hex: String, + /// Content findings; empty on success. + pub findings: Vec, +} + +/// Successful restore into a fresh workspace root. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct BackupRestoreReport { + /// Report schema. + pub schema: &'static str, + /// Fresh workspace root published atomically. + pub dest: String, + /// Restored workspace id. + pub workspace_id: String, + /// Journal files restored. + pub store_file_count: usize, + /// Journal bytes restored. + pub store_bytes: u64, + /// Verified source manifest digest. + pub manifest_hash_hex: String, + /// Whether the restored `BatPak` chain verified before publication. + pub chain_verified: bool, +} diff --git a/src/backup/restore.rs b/src/backup/restore.rs new file mode 100644 index 0000000..c20c2ec --- /dev/null +++ b/src/backup/restore.rs @@ -0,0 +1,203 @@ +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use batpak::store::{ReadOnly, Store, StoreConfig}; + +use crate::error::TexoError; + +use super::filesystem::{ + absolute_path, backup_error, hash_regular_bounded, read_regular_bounded, safe_flat_name, + sync_directory, write_new_synced, +}; +use super::verify::verify_with_expected_manifest_hash; +use super::{ + BackupManifest, BackupRestoreReport, FileRecord, CONFIG_FILE, MANIFEST_FILE, + MAX_MANIFEST_BYTES, MAX_STORE_FILE_BYTES, STORE_DIR, +}; + +static RESTORE_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Restore a verified backup into a fresh workspace root. +/// +/// The destination must not exist. Restore verifies the source first, copies +/// only manifest-listed regular files into a private sibling staging root, +/// rewrites the selected workspace store path to a safe root-relative default, +/// verifies the copied `BatPak` chain, and atomically renames the staging root +/// into place. Derived caches and client integration are intentionally absent. +/// +/// # Errors +/// Returns an error for an invalid backup, overlapping/existing destination, +/// unsafe source bytes, copied-file mismatch, or failed restored chain. +pub fn restore( + backup: &Path, + dest: &Path, + expected_manifest_hash: Option<&str>, +) -> Result { + let verification = verify_with_expected_manifest_hash(backup, expected_manifest_hash)?; + if !verification.verified { + return Err(backup_error(format!( + "backup restore refused: {}", + verification + .findings + .iter() + .map(|finding| finding.kind) + .collect::>() + .join(", ") + ))); + } + let backup = absolute_path(backup)?; + let dest = absolute_path(dest)?; + if dest.starts_with(&backup) || backup.starts_with(&dest) { + return Err(backup_error( + "restore destination must not overlap the backup", + )); + } + let manifest_bytes = read_regular_bounded(&backup.join(MANIFEST_FILE), MAX_MANIFEST_BYTES) + .map_err(backup_error)?; + let manifest: BackupManifest = serde_json::from_slice(&manifest_bytes)?; + let source_config = crate::config::TexoRootConfig::load(&backup.join(CONFIG_FILE)) + .map_err(|error| backup_error(error.to_string()))?; + let mut workspace = source_config + .workspaces + .get(&manifest.workspace_id) + .cloned() + .ok_or_else(|| backup_error("backup config does not contain its workspace"))?; + let restore_store_path = crate::config::WorkspaceEntry::for_id(&manifest.workspace_id) + .primary() + .map_err(|error| backup_error(error.to_string()))? + .store_path; + workspace + .set_primary_store_path(restore_store_path.clone()) + .map_err(|error| backup_error(error.to_string()))?; + let mut workspaces = std::collections::BTreeMap::new(); + workspaces.insert(manifest.workspace_id.clone(), workspace.clone()); + let restored_config = crate::config::TexoRootConfig { + default_workspace: manifest.workspace_id.clone(), + workspaces, + gateway: source_config.gateway, + }; + let destination = RestoreDestination::create(&dest)?; + let texo_dir = destination.path().join(".texo"); + fs::create_dir(&texo_dir)?; + let config_bytes = toml::to_string_pretty(&restored_config) + .map_err(|error| backup_error(error.to_string()))?; + write_new_synced(&texo_dir.join(CONFIG_FILE), config_bytes.as_bytes())?; + let store_dest = destination.path().join(&restore_store_path); + fs::create_dir_all(&store_dest)?; + copy_verified_store(&backup.join(STORE_DIR), &store_dest, &manifest.store_files)?; + let store = Store::::open_read_only(StoreConfig::new(&store_dest))?; + let chain = store.verify_chain()?; + if !chain.is_intact() { + return Err(backup_error(format!( + "restored store chain verification failed: {chain:?}" + ))); + } + drop(store); + sync_directory(&store_dest)?; + sync_directory(&texo_dir)?; + destination.complete()?; + Ok(BackupRestoreReport { + schema: "texo.backup-restore.v1", + dest: dest.display().to_string(), + workspace_id: manifest.workspace_id, + store_file_count: manifest.store_files.len(), + store_bytes: manifest.store_files.iter().map(|record| record.bytes).sum(), + manifest_hash_hex: verification.manifest_hash_hex, + chain_verified: true, + }) +} + +fn copy_verified_store( + source: &Path, + dest: &Path, + records: &[FileRecord], +) -> Result<(), TexoError> { + for record in records { + if !safe_flat_name(&record.name) { + return Err(backup_error("unsafe store record during restore")); + } + let target = dest.join(&record.name); + fs::copy(source.join(&record.name), &target)?; + File::open(&target)?.sync_all()?; + let (hash, bytes) = + hash_regular_bounded(&target, MAX_STORE_FILE_BYTES).map_err(backup_error)?; + if hash != record.hash_hex || bytes != record.bytes { + return Err(backup_error(format!( + "restored copy of {} differs from verified source", + record.name + ))); + } + } + Ok(()) +} + +struct RestoreDestination { + final_path: PathBuf, + stage_path: PathBuf, + complete: bool, +} + +impl RestoreDestination { + fn create(dest: &Path) -> Result { + match fs::symlink_metadata(dest) { + Ok(_) => return Err(backup_error("restore destination already exists")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let parent = dest + .parent() + .ok_or_else(|| backup_error("restore destination has no parent"))?; + fs::create_dir_all(parent)?; + for _attempt in 0..100 { + let sequence = RESTORE_COUNTER.fetch_add(1, Ordering::Relaxed); + let stage_path = + parent.join(format!(".texo-restore-{}-{sequence}", std::process::id())); + let created = { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt as _; + builder.mode(0o700); + } + builder.create(&stage_path) + }; + match created { + Ok(()) => { + return Ok(Self { + final_path: dest.to_path_buf(), + stage_path, + complete: false, + }); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + Err(backup_error( + "could not allocate a private restore staging root", + )) + } + + fn path(&self) -> &Path { + &self.stage_path + } + + fn complete(mut self) -> Result<(), TexoError> { + sync_directory(&self.stage_path)?; + fs::rename(&self.stage_path, &self.final_path)?; + if let Some(parent) = self.final_path.parent() { + sync_directory(parent)?; + } + self.complete = true; + Ok(()) + } +} + +impl Drop for RestoreDestination { + fn drop(&mut self) { + if !self.complete { + let _ignored = fs::remove_dir_all(&self.stage_path); + } + } +} diff --git a/src/backup/verify.rs b/src/backup/verify.rs new file mode 100644 index 0000000..f41736e --- /dev/null +++ b/src/backup/verify.rs @@ -0,0 +1,484 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use batpak::store::backup_envelope::{ + backup_manifest_body_hash, restore_proof_evidence_report, BackupSegmentRef, + BACKUP_MANIFEST_BODY_SCHEMA_VERSION, +}; +use batpak::store::{ + snapshot_report_body_hash, ReadOnly, Store, StoreConfig, + SNAPSHOT_EVIDENCE_REPORT_SCHEMA_VERSION, +}; + +use crate::error::TexoError; + +use super::filesystem::{ + absolute_path, backup_error, digest_from_hex, finding, hash_regular_bounded, + read_regular_bounded, safe_flat_name, +}; +use super::{ + BackupFinding, BackupManifest, BackupVerifyReport, FileRecord, CONFIG_FILE, MANIFEST_FILE, + MANIFEST_SCHEMA, MAX_CONFIG_BYTES, MAX_MANIFEST_BYTES, MAX_STORE_FILES, MAX_STORE_FILE_BYTES, + STORE_DIR, +}; + +static VERIFY_COPY_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Verify a backup using only bytes beneath its destination. +/// +/// Content failures are findings, not function errors. Function errors are +/// reserved for environmental failures such as unreadable directory entries. +/// +/// # Errors +/// Returns an error only when the destination cannot be safely inspected. +pub fn verify(dest: &Path) -> Result { + verify_with_expected_manifest_hash(dest, None) +} + +/// Verify a backup and optionally compare its manifest to an out-of-band pin. +/// +/// # Errors +/// Returns an input error for a malformed expected digest, or an environment +/// error when the destination cannot be safely inspected. +pub fn verify_with_expected_manifest_hash( + dest: &Path, + expected_manifest_hash: Option<&str>, +) -> Result { + if let Some(expected) = expected_manifest_hash { + if expected.len() != 64 || !expected.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(TexoError::OpInput { + op: "texo backup verify".to_string(), + detail: "expected manifest hash must be exactly 64 hexadecimal characters" + .to_string(), + }); + } + } + let original = dest.to_path_buf(); + let metadata = match fs::symlink_metadata(dest) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(empty_report( + &original, + finding("backup_missing", "backup destination does not exist"), + )); + } + Err(error) => return Err(error.into()), + }; + if !metadata.file_type().is_dir() { + return Ok(empty_report( + &original, + finding( + "backup_root_invalid", + "backup root must be a regular directory", + ), + )); + } + let dest = absolute_path(dest)?; + let manifest_path = dest.join(MANIFEST_FILE); + let manifest_bytes = match read_regular_bounded(&manifest_path, MAX_MANIFEST_BYTES) { + Ok(bytes) => bytes, + Err(detail) => return Ok(empty_report(&dest, finding("manifest_invalid", detail))), + }; + let manifest_hash_hex = blake3::hash(&manifest_bytes).to_hex().to_string(); + let manifest: BackupManifest = match serde_json::from_slice(&manifest_bytes) { + Ok(manifest) => manifest, + Err(error) => { + return Ok(empty_report_with_hash( + &dest, + manifest_hash_hex, + finding("manifest_invalid", error.to_string()), + )); + } + }; + let mut findings = Vec::new(); + if expected_manifest_hash.is_some_and(|expected| expected != manifest_hash_hex) { + findings.push(finding( + "manifest_hash_mismatch", + format!("manifest hash is {manifest_hash_hex}; it does not match the out-of-band pin"), + )); + } + if manifest.schema != MANIFEST_SCHEMA { + findings.push(finding( + "manifest_schema_unsupported", + format!("unsupported schema `{}`", manifest.schema), + )); + } + if manifest.store_files.len() > MAX_STORE_FILES { + findings.push(finding( + "manifest_file_limit", + format!("manifest records more than {MAX_STORE_FILES} store files"), + )); + } + check_top_level(&dest, &mut findings)?; + check_snapshot_evidence(&dest, &manifest, &mut findings); + check_substrate_restore_proof(&dest, &manifest, &mut findings); + let store_files_valid = check_store_files(&dest, &manifest, &mut findings)?; + let config_valid = check_config(&dest, &manifest, &mut findings); + if config_valid { + check_config_binding(&dest, &manifest, &mut findings); + } + if store_files_valid == manifest.store_files.len() { + check_store_read_only(&dest, &manifest, &mut findings); + } + Ok(BackupVerifyReport { + schema: "texo.backup-verify.v1", + verified: findings.is_empty(), + dest: dest.display().to_string(), + workspace_id: manifest.workspace_id, + store_files_valid, + store_files_expected: manifest.store_files.len(), + manifest_hash_hex, + findings, + }) +} + +fn check_substrate_restore_proof( + dest: &Path, + manifest: &BackupManifest, + findings: &mut Vec, +) { + if manifest.substrate_manifest.schema_version != BACKUP_MANIFEST_BODY_SCHEMA_VERSION { + findings.push(finding( + "substrate_manifest_schema_unsupported", + format!( + "unsupported substrate manifest schema {}", + manifest.substrate_manifest.schema_version + ), + )); + } + if manifest.substrate_manifest.backup_id != manifest.snapshot.body.snapshot_id { + findings.push(finding( + "substrate_manifest_snapshot_mismatch", + "substrate backup identity does not match snapshot identity", + )); + } + let claimed_hash = digest_from_hex(&manifest.substrate_manifest_hash_hex); + match backup_manifest_body_hash(&manifest.substrate_manifest) { + Ok(computed) if claimed_hash == Some(computed) => {} + Ok(_) => findings.push(finding( + "substrate_manifest_hash_mismatch", + "substrate manifest body hash does not recompute", + )), + Err(error) => findings.push(finding( + "substrate_manifest_invalid", + format!("substrate manifest cannot be encoded: {error}"), + )), + } + let mut observed = Vec::new(); + let store_dir = dest.join(STORE_DIR); + let entries = match fs::read_dir(&store_dir) { + Ok(entries) => entries, + Err(error) => { + findings.push(finding( + "substrate_restore_proof_invalid", + error.to_string(), + )); + return; + } + }; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + findings.push(finding( + "substrate_restore_proof_invalid", + error.to_string(), + )); + return; + } + }; + let name = entry.file_name().to_string_lossy().to_string(); + let path = Path::new(&name); + if path.extension() != Some(std::ffi::OsStr::new("fbat")) { + continue; + } + let Some(stem) = path.file_stem().and_then(std::ffi::OsStr::to_str) else { + continue; + }; + let segment_id = match stem.parse::() { + Ok(segment_id) if format!("{segment_id:06}.fbat") == name => segment_id, + _ => { + findings.push(finding( + "substrate_restore_proof_invalid", + format!("invalid segment filename `{name}`"), + )); + continue; + } + }; + match hash_regular_bounded(&entry.path(), MAX_STORE_FILE_BYTES) { + Ok((hash, _bytes)) => match digest_from_hex(&hash) { + Some(bytes_digest) => observed.push(BackupSegmentRef { + segment_id, + bytes_digest, + }), + None => findings.push(finding( + "substrate_restore_proof_invalid", + format!("invalid digest for `{name}`"), + )), + }, + Err(detail) => findings.push(finding("substrate_restore_proof_invalid", detail)), + } + } + match restore_proof_evidence_report(&manifest.substrate_manifest, &observed) { + Ok(report) if report.body.findings.is_empty() => {} + Ok(report) => findings.push(finding( + "substrate_restore_proof_failed", + format!("BatPak restore proof findings: {:?}", report.body.findings), + )), + Err(error) => findings.push(finding( + "substrate_restore_proof_invalid", + format!("BatPak restore proof cannot be encoded: {error}"), + )), + } +} + +fn check_snapshot_evidence( + dest: &Path, + manifest: &BackupManifest, + findings: &mut Vec, +) { + if manifest.snapshot.body.schema_version != SNAPSHOT_EVIDENCE_REPORT_SCHEMA_VERSION { + findings.push(finding( + "snapshot_schema_unsupported", + format!( + "unsupported snapshot schema {}", + manifest.snapshot.body.schema_version + ), + )); + } + match snapshot_report_body_hash(&manifest.snapshot.body) { + Ok(hash) if hash == manifest.snapshot.body_hash => {} + Ok(_) => findings.push(finding( + "snapshot_evidence_mismatch", + "snapshot evidence body hash does not recompute", + )), + Err(error) => findings.push(finding( + "snapshot_evidence_mismatch", + format!("snapshot evidence cannot be encoded: {error}"), + )), + } + let path_hash = blake3::hash(dest.join(STORE_DIR).as_os_str().as_encoded_bytes()); + if path_hash.as_bytes() != &manifest.snapshot.body.destination_path_digest { + findings.push(finding( + "snapshot_destination_mismatch", + "snapshot evidence is bound to a different destination path", + )); + } +} + +fn check_store_files( + dest: &Path, + manifest: &BackupManifest, + findings: &mut Vec, +) -> Result { + let store_dir = dest.join(STORE_DIR); + match fs::symlink_metadata(&store_dir) { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => { + findings.push(finding( + "store_directory_invalid", + "store must be a regular directory, not a symbolic link or file", + )); + return Ok(0); + } + Err(error) => { + findings.push(finding("store_directory_invalid", error.to_string())); + return Ok(0); + } + } + let mut expected = std::collections::BTreeSet::new(); + let mut valid = 0; + for record in &manifest.store_files { + if !safe_flat_name(&record.name) || !expected.insert(record.name.clone()) { + findings.push(finding( + "store_record_invalid", + format!("unsafe or duplicate store file `{}`", record.name), + )); + continue; + } + let path = store_dir.join(&record.name); + match hash_regular_bounded(&path, MAX_STORE_FILE_BYTES) { + Ok((hash, bytes)) if hash == record.hash_hex && bytes == record.bytes => valid += 1, + Ok((hash, bytes)) => findings.push(finding( + "store_file_mismatch", + format!( + "{} expected {} bytes/{} but found {bytes}/{hash}", + record.name, record.bytes, record.hash_hex + ), + )), + Err(detail) => findings.push(finding("store_file_invalid", detail)), + } + } + match fs::read_dir(&store_dir) { + Ok(entries) => { + for entry in entries { + let entry = entry?; + let name = entry.file_name().to_string_lossy().to_string(); + if !expected.contains(&name) { + findings.push(finding( + "unexpected_store_file", + format!("unrecorded store entry `{name}`"), + )); + } + } + } + Err(error) => findings.push(finding("store_directory_invalid", error.to_string())), + } + Ok(valid) +} + +fn check_config(dest: &Path, manifest: &BackupManifest, findings: &mut Vec) -> bool { + match hash_regular_bounded(&dest.join(CONFIG_FILE), MAX_CONFIG_BYTES) { + Ok((hash, bytes)) if hash == manifest.config_hash_hex && bytes == manifest.config_bytes => { + true + } + Ok((hash, bytes)) => { + findings.push(finding( + "config_mismatch", + format!( + "expected {}/{} but found {bytes}/{hash}", + manifest.config_bytes, manifest.config_hash_hex + ), + )); + false + } + Err(detail) => { + findings.push(finding("config_invalid", detail)); + false + } + } +} + +fn check_config_binding(dest: &Path, manifest: &BackupManifest, findings: &mut Vec) { + let config = match crate::config::TexoRootConfig::load(&dest.join(CONFIG_FILE)) { + Ok(config) => config, + Err(error) => { + findings.push(finding("config_binding_invalid", error.to_string())); + return; + } + }; + match config.resolve(Some(&manifest.workspace_id)) { + Ok(workspace) if workspace.store_path == manifest.store_path => {} + Ok(workspace) => findings.push(finding( + "config_binding_mismatch", + format!( + "manifest store path `{}` differs from config `{}`", + manifest.store_path, workspace.store_path + ), + )), + Err(error) => findings.push(finding("config_binding_mismatch", error.to_string())), + } +} + +fn check_store_read_only( + dest: &Path, + manifest: &BackupManifest, + findings: &mut Vec, +) { + let copy = match VerificationCopy::create(&dest.join(STORE_DIR), &manifest.store_files) { + Ok(copy) => copy, + Err(error) => { + findings.push(finding( + "store_open_invalid", + format!("snapshot cannot be prepared for read-only verification: {error}"), + )); + return; + } + }; + match Store::::open_read_only(StoreConfig::new(copy.path())) { + Ok(store) => match store.verify_chain() { + Ok(chain) if chain.is_intact() => {} + Ok(chain) => findings.push(finding( + "store_chain_invalid", + format!("snapshot chain verification failed: {chain:?}"), + )), + Err(error) => findings.push(finding( + "store_chain_invalid", + format!("snapshot chain verification errored: {error}"), + )), + }, + Err(error) => findings.push(finding( + "store_open_invalid", + format!("snapshot cannot open read-only: {error}"), + )), + } +} + +struct VerificationCopy { + path: PathBuf, +} + +impl VerificationCopy { + fn create(source: &Path, records: &[FileRecord]) -> Result { + let parent = std::env::temp_dir(); + for _attempt in 0..100 { + let counter = VERIFY_COPY_COUNTER.fetch_add(1, Ordering::Relaxed); + let path = parent.join(format!( + "texo-backup-verify-{}-{counter}", + std::process::id() + )); + match fs::create_dir(&path) { + Ok(()) => { + let copy = Self { path }; + for record in records { + if !safe_flat_name(&record.name) { + return Err(backup_error("unsafe store record in verification copy")); + } + fs::copy(source.join(&record.name), copy.path.join(&record.name))?; + } + return Ok(copy); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + Err(backup_error("could not allocate verification directory")) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for VerificationCopy { + fn drop(&mut self) { + let _ignored = fs::remove_dir_all(&self.path); + } +} + +fn check_top_level(dest: &Path, findings: &mut Vec) -> Result<(), TexoError> { + let expected = std::collections::BTreeSet::from([MANIFEST_FILE, CONFIG_FILE, STORE_DIR]); + for entry in fs::read_dir(dest)? { + let entry = entry?; + let name = entry.file_name().to_string_lossy().to_string(); + if !expected.contains(name.as_str()) { + findings.push(finding( + "unexpected_backup_entry", + format!("unrecorded top-level entry `{name}`"), + )); + } + } + Ok(()) +} + +fn empty_report(dest: &Path, finding: BackupFinding) -> BackupVerifyReport { + empty_report_with_hash(dest, String::new(), finding) +} + +fn empty_report_with_hash( + dest: &Path, + manifest_hash_hex: String, + finding: BackupFinding, +) -> BackupVerifyReport { + BackupVerifyReport { + schema: "texo.backup-verify.v1", + verified: false, + dest: dest.display().to_string(), + workspace_id: String::new(), + store_files_valid: 0, + store_files_expected: 0, + manifest_hash_hex, + findings: vec![finding], + } +} diff --git a/src/bin/texo.rs b/src/bin/texo.rs index 0060eb7..e8d9701 100644 --- a/src/bin/texo.rs +++ b/src/bin/texo.rs @@ -1,8 +1,8 @@ //! texo CLI entrypoint. +use std::io::Write as _; use std::process::ExitCode; -#[expect(clippy::print_stderr, reason = "CLI output contract")] fn main() -> ExitCode { tracing_subscriber::fmt() .with_env_filter( @@ -13,14 +13,14 @@ fn main() -> ExitCode { .init(); if let Err(error) = batpak::event::validate_event_payload_registry() { - eprintln!("{error}"); + let _rendered = writeln!(std::io::stderr().lock(), "{error}"); return ExitCode::FAILURE; } match texo::surfaces::cli::run() { Ok(code) => code, Err(error) => { - texo::surfaces::cli::render::cli_error(&error); + let _rendered = texo::surfaces::cli::render::cli_error(&error); ExitCode::FAILURE } } diff --git a/src/claims/campaign.rs b/src/claims/campaign.rs new file mode 100644 index 0000000..afd93d0 --- /dev/null +++ b/src/claims/campaign.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; + +use crate::events::payloads::RelationCampaignCheckpointV1; + +/// Latest durable progress proof for one workspace relation campaign. +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, batpak::EventSourced)] +#[batpak(input = batpak::event::RawMsgpackInput, cache_version = 1, state_max_cardinality = 1)] +#[batpak(event = RelationCampaignCheckpointV1, handler = on_checkpoint)] +pub struct CampaignCard { + /// Last checkpoint in journal commit order. + pub latest: Option, +} + +impl CampaignCard { + fn on_checkpoint(&mut self, event: &RelationCampaignCheckpointV1) { + self.latest = Some(event.clone()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::ids::WorkspaceId; + use crate::relate::settlement::CampaignPhase; + + fn checkpoint(phase: CampaignPhase, observed_at_ms: u64) -> RelationCampaignCheckpointV1 { + RelationCampaignCheckpointV1 { + workspace_id: WorkspaceId::try_from("workspace").expect("valid workspace"), + evaluated_basis_digest_hex: "a".repeat(64), + result_basis_digest_hex: "a".repeat(64), + candidate_policy_digest_hex: "b".repeat(64), + phase, + observed_at_ms, + } + } + + #[test] + fn latest_checkpoint_wins_in_event_order() { + let partial = checkpoint( + CampaignPhase::Partial { + next_candidate_cursor: 10, + }, + 20, + ); + let complete = checkpoint(CampaignPhase::Complete, 10); + let mut card = CampaignCard::default(); + card.on_checkpoint(&partial); + card.on_checkpoint(&complete); + assert_eq!(card.latest, Some(complete)); + } +} diff --git a/src/claims/mod.rs b/src/claims/mod.rs index af63915..83b70ef 100644 --- a/src/claims/mod.rs +++ b/src/claims/mod.rs @@ -1,5 +1,6 @@ //! Claim-domain projections and workspace views. +pub(crate) mod campaign; pub mod card; pub mod compile_log; pub mod conflict; diff --git a/src/claims/status.rs b/src/claims/status.rs index ac893ea..e024a68 100644 --- a/src/claims/status.rs +++ b/src/claims/status.rs @@ -20,6 +20,7 @@ pub enum ClaimStatus { impl ClaimStatus { /// Serialize to the old status string form. + #[must_use] pub const fn as_str(self) -> &'static str { match self { Self::Current => "current", @@ -49,6 +50,7 @@ pub enum ConflictStatus { impl ConflictStatus { /// Serialize to the old status string form. + #[must_use] pub const fn as_str(self) -> &'static str { match self { Self::Open => "open", @@ -65,6 +67,7 @@ impl fmt::Display for ConflictStatus { } /// Derive claim status with Superseded > Conflicting > Current precedence. +#[must_use] pub fn claim_status(card: &ClaimCard, in_open_conflict: bool) -> ClaimStatus { if card.phase == 2 { ClaimStatus::Superseded diff --git a/src/code_index.rs b/src/code_index.rs index b8696d2..268912b 100644 --- a/src/code_index.rs +++ b/src/code_index.rs @@ -1,1019 +1,17 @@ //! Bounded SCIP import and built-in code-intelligence fallbacks. -use std::collections::{BTreeMap, BTreeSet}; -use std::io::Write as _; -use std::ops::ControlFlow; -use std::path::{Component, Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use protobuf::Message; - -use crate::error::TexoError; -use crate::events::ids::blake3_bytes_hex; -use crate::git_source::{CapturedSource, GitCapture}; -use crate::knowledge::{ - AnalysisQuality, ByteRange, CodeIndexArtifact, CodeIndexFormat, CodeIndexId, CodeOccurrence, - CodeOccurrenceRole, CoverageGap, CoverageGapKind, KnowledgeCoverage, LineRange, - MAX_EVIDENCE_EXCERPT_BYTES, -}; - /// Normalized artifact schema. pub const ARTIFACT_SCHEMA: &str = "texo.code-index.v3"; -const SCIP_DEFINITION: i32 = 0x1; -const SCIP_IMPORT: i32 = 0x2; -const SCIP_WRITE: i32 = 0x4; -const SCIP_READ: i32 = 0x8; -const SCIP_GENERATED: i32 = 0x10; -const SCIP_TEST: i32 = 0x20; -const SCIP_FORWARD_DEFINITION: i32 = 0x40; -const MAX_GAPS: usize = 256; -const MAX_LEXICAL_OCCURRENCES_PER_SOURCE: usize = 512; -static ARTIFACT_TMP_COUNTER: AtomicU64 = AtomicU64::new(0); - -/// Bounds for one code-index build/import. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CodeIndexLimits { - /// Maximum raw SCIP bytes accepted. - pub max_scip_bytes: u64, - /// Maximum documents consumed. - pub max_documents: usize, - /// Maximum normalized occurrences retained. - pub max_occurrences: usize, - /// Global wall budget for built-in analysis. - pub analysis_budget: Duration, -} - -impl Default for CodeIndexLimits { - fn default() -> Self { - Self { - max_scip_bytes: 64 * 1024 * 1024, - max_documents: 20_000, - max_occurrences: 200_000, - analysis_budget: Duration::from_secs(30), - } - } -} - -/// A built artifact and the digest of its serialized bytes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PreparedCodeIndex { - /// Normalized disposable artifact. - pub artifact: CodeIndexArtifact, - /// BLAKE3 digest of the exact persisted artifact bytes. - pub artifact_digest_hex: String, - /// Serialized artifact bytes. - pub bytes: Vec, -} - -/// Read a workspace-local SCIP file with a hard byte bound. -/// -/// # Errors -/// Fails for paths outside the workspace, symlinks, non-regular files, and -/// files exceeding the declared bound. -pub fn read_scip(root: &Path, path: &Path, max_bytes: u64) -> Result, TexoError> { - let candidate = if path.is_absolute() { - path.to_path_buf() - } else { - root.join(path) - }; - let canonical_root = std::fs::canonicalize(root)?; - let canonical = std::fs::canonicalize(&candidate)?; - if !canonical.starts_with(&canonical_root) { - return Err(source_error(path, "SCIP path escapes the workspace")); - } - let metadata = std::fs::symlink_metadata(&candidate)?; - if metadata.file_type().is_symlink() || !metadata.is_file() { - return Err(source_error( - path, - "SCIP input must be a regular non-symlink file", - )); - } - if metadata.len() > max_bytes { - return Err(source_error( - path, - "SCIP input exceeds the configured byte limit", - )); - } - let bytes = std::fs::read(&candidate)?; - if u64::try_from(bytes.len()).unwrap_or(u64::MAX) != metadata.len() { - return Err(source_error(path, "SCIP input changed while it was read")); - } - Ok(bytes) -} - -/// Build a deterministic code index for one frozen Git capture. -/// -/// SCIP occurrences are preferred. Sources absent from the imported SCIP -/// index fall back to the built-in Rust grammar or bounded lexical discovery. -/// -/// # Errors -/// Returns a typed source error for malformed SCIP, invalid source ranges, or -/// artifact serialization failure. -pub fn build( - capture: &GitCapture, - scip_bytes: Option<&[u8]>, - limits: CodeIndexLimits, -) -> Result { - let source_map = capture - .sources - .iter() - .map(|source| (source.path.as_str(), source)) - .collect::>(); - let mut builder = ArtifactBuilder::new(limits, capture.coverage.clone()); - let mut indexed_paths = BTreeSet::new(); - let mut analyzer_parts = Vec::new(); - if let Some(bytes) = scip_bytes { - let before = builder.occurrences.len(); - let analyzer = import_scip(bytes, &source_map, &mut builder, &mut indexed_paths)?; - analyzer_parts.push(analyzer); - // Only claim compiler-precise coverage when SCIP actually contributed - // occurrences. An empty index, all-missing documents, or all-skipped - // ranges must not promote lexical/syntactic rows to `precise`. - if builder.occurrences.len() > before { - builder.format = CodeIndexFormat::Scip; - builder.quality = AnalysisQuality::Precise; - } - } - let fallback = analyze_fallbacks( - &capture.sources, - &indexed_paths, - &mut builder, - limits.analysis_budget, - )?; - if !fallback.is_empty() { - analyzer_parts.push(fallback); - } - if analyzer_parts.is_empty() { - analyzer_parts.push("texo-lexical:v2".to_string()); - } - let analyzer_fingerprint = analyzer_parts.join("+"); - let mut occurrences = builder.occurrences; - occurrences.sort_by(code_occurrence_order); - occurrences.dedup(); - let occurrence_material = batpak::encoding::to_bytes(&occurrences) - .map_err(|error| source_error(Path::new(".texo/cache/code-index"), &error.to_string()))?; - // Derive identity from the normalized occurrences actually persisted, not the - // raw SCIP bytes: two builds of the same snapshot+SCIP under different limits - // truncate differently, and the id must track what lands on disk so the cache - // never serves a digest-mismatched or silently truncated artifact. - let raw_digest = blake3_bytes_hex(&occurrence_material); - let index_id = CodeIndexId::derive(&format!( - "{ARTIFACT_SCHEMA}\u{1f}{}\u{1f}{raw_digest}\u{1f}{analyzer_fingerprint}", - capture.snapshot_id - )); - let coverage = KnowledgeCoverage { - analysis_quality: builder.quality, - sources_examined: builder.sources_examined, - occurrences: u64::try_from(occurrences.len()).unwrap_or(u64::MAX), - truncated: builder.truncated, - gaps: builder.gaps, - }; - let artifact = CodeIndexArtifact { - schema: ARTIFACT_SCHEMA.to_string(), - snapshot_id: capture.snapshot_id.clone(), - index_id, - format: builder.format, - analyzer_fingerprint, - occurrences, - coverage, - }; - let bytes = batpak::encoding::to_bytes(&artifact) - .map_err(|error| source_error(Path::new(".texo/cache/code-index"), &error.to_string()))?; - let artifact_digest_hex = blake3_bytes_hex(&bytes); - Ok(PreparedCodeIndex { - artifact, - artifact_digest_hex, - bytes, - }) -} - -/// Persist a normalized artifact with atomic content-addressed replacement. -/// -/// # Errors -/// Returns an I/O error for staging, flush, rename, or directory sync failure. -pub fn persist(root: &Path, prepared: &PreparedCodeIndex) -> Result { - let path = artifact_path(root, &prepared.artifact.index_id); - atomic_write(&path, &prepared.bytes)?; - Ok(path) -} - -/// Load and authenticate one disposable normalized code index. -/// -/// Missing artifacts return `Ok(None)` so callers can report degraded -/// coverage. Present but malformed or digest-mismatched artifacts fail closed. -/// -/// # Errors -/// Returns a typed decode/source error when a present artifact is invalid. -pub fn load( - root: &Path, - index_id: &CodeIndexId, - expected_digest_hex: &str, -) -> Result, TexoError> { - let path = artifact_path(root, index_id); - let bytes = match std::fs::read(&path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), - }; - if blake3_bytes_hex(&bytes) != expected_digest_hex { - return Err(source_error(&path, "code-index artifact digest mismatch")); - } - let artifact = batpak::encoding::from_bytes::(&bytes).map_err(|error| { - TexoError::Decode { - entity: index_id.to_string(), - detail: error.to_string(), - } - })?; - if artifact.schema != ARTIFACT_SCHEMA { - return Ok(None); - } - if artifact.index_id != *index_id { - return Err(source_error(&path, "code-index artifact identity mismatch")); - } - Ok(Some(artifact)) -} - -fn artifact_path(root: &Path, index_id: &CodeIndexId) -> PathBuf { - root.join(".texo") - .join("cache") - .join("code-index") - .join(format!("{}.bin", index_id.as_str())) -} - -struct ArtifactBuilder { - limits: CodeIndexLimits, - occurrences: Vec, - sources_examined: u64, - truncated: bool, - gaps: Vec, - format: CodeIndexFormat, - quality: AnalysisQuality, -} - -impl ArtifactBuilder { - fn new(limits: CodeIndexLimits, source_coverage: KnowledgeCoverage) -> Self { - Self { - limits, - occurrences: Vec::new(), - sources_examined: 0, - truncated: source_coverage.truncated, - gaps: source_coverage.gaps, - format: CodeIndexFormat::Lexical, - quality: AnalysisQuality::Lexical, - } - } - - fn push(&mut self, occurrence: CodeOccurrence) -> bool { - if self.occurrences.len() >= self.limits.max_occurrences { - self.truncated = true; - self.gap(None, CoverageGapKind::BudgetExceeded); - return false; - } - self.occurrences.push(occurrence); - true - } - - fn gap(&mut self, path: Option, kind: CoverageGapKind) { - if self.gaps.len() < MAX_GAPS { - let gap = CoverageGap { path, kind }; - if !self.gaps.contains(&gap) { - self.gaps.push(gap); - } - } else { - self.truncated = true; - } - } -} - -fn import_scip( - bytes: &[u8], - sources: &BTreeMap<&str, &CapturedSource>, - builder: &mut ArtifactBuilder, - indexed_paths: &mut BTreeSet, -) -> Result { - let index = scip::types::Index::parse_from_bytes(bytes) - .map_err(|error| source_error(Path::new("index.scip"), &error.to_string()))?; - let analyzer = scip_analyzer_fingerprint(&index); - for document in index.documents.iter().take(builder.limits.max_documents) { - if !safe_relative_path(&document.relative_path) { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - continue; - } - builder.sources_examined = builder.sources_examined.saturating_add(1); - let Some(source) = sources.get(document.relative_path.as_str()).copied() else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::MissingObject, - ); - continue; - }; - if !position_is_utf8(document) { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - } - if std::str::from_utf8(&source.bytes).is_err() { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - } - let before = builder.occurrences.len(); - import_scip_document(document, source, &analyzer, builder); - // Only claim the path as SCIP-indexed when the document actually produced - // occurrences. A matching-but-empty document must fall through to the - // built-in fallback rather than suppress every symbol for that file. - if builder.occurrences.len() > before { - indexed_paths.insert(document.relative_path.clone()); - } - } - if index.documents.len() > builder.limits.max_documents { - builder.truncated = true; - builder.gap(None, CoverageGapKind::BudgetExceeded); - } - Ok(analyzer) -} - -fn scip_analyzer_fingerprint(index: &scip::types::Index) -> String { - let metadata = index.metadata.as_ref(); - let tool = metadata.and_then(|metadata| metadata.tool_info.as_ref()); - let name = tool.map_or("unknown", |tool| tool.name.as_str()); - let version = tool.map_or("unknown", |tool| tool.version.as_str()); - format!("scip:{name}:{version}:protocol-v0") -} - -fn position_is_utf8(document: &scip::types::Document) -> bool { - document.position_encoding.enum_value().ok() - == Some(scip::types::PositionEncoding::UTF8CodeUnitOffsetFromLineStart) -} - -fn import_scip_document( - document: &scip::types::Document, - source: &CapturedSource, - analyzer: &str, - builder: &mut ArtifactBuilder, -) { - let lines = line_offsets(&source.bytes); - let source_digest_hex = blake3_bytes_hex(&source.bytes); - for occurrence in &document.occurrences { - if occurrence.symbol.is_empty() { - continue; - } - let Some(range) = scip_range(occurrence) else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - continue; - }; - let Some((byte_range, line_range)) = resolve_range(&lines, &source.bytes, range) else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - continue; - }; - let (Ok(start), Ok(end)) = ( - usize::try_from(byte_range.start), - usize::try_from(byte_range.end), - ) else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - continue; - }; - let excerpt_bytes = &source.bytes[start..end]; - let Ok(excerpt) = std::str::from_utf8(excerpt_bytes) else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - }; - if excerpt.len() > MAX_EVIDENCE_EXCERPT_BYTES { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::SourceTooLarge, - ); - continue; - } - let roles = scip_roles(occurrence.symbol_roles); - let display_name = if excerpt.is_empty() { - occurrence.symbol.clone() - } else { - excerpt.to_string() - }; - let Some((context, context_byte_range, context_line_range)) = - source_context(&source.bytes, start, end) - else { - builder.gap( - Some(document.relative_path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - }; - if !builder.push(CodeOccurrence { - symbol: occurrence.symbol.clone(), - display_name, - roles, - path: document.relative_path.clone(), - byte_range, - line_range, - source_digest_hex: source_digest_hex.clone(), - excerpt: excerpt.to_string(), - context, - context_byte_range, - context_line_range, - analyzer_fingerprint: analyzer.to_string(), - analysis_quality: AnalysisQuality::Precise, - }) { - break; - } - } -} - -#[derive(Debug, Clone, Copy)] -struct ZeroRange { - start_line: usize, - start_character: usize, - end_line: usize, - end_character: usize, -} - -fn scip_range(occurrence: &scip::types::Occurrence) -> Option { - use scip::types::occurrence::Typed_range; - match occurrence.typed_range.as_ref() { - Some(Typed_range::SingleLineRange(range)) => Some(ZeroRange { - start_line: nonnegative(range.line)?, - start_character: nonnegative(range.start_character)?, - end_line: nonnegative(range.line)?, - end_character: nonnegative(range.end_character)?, - }), - Some(Typed_range::MultiLineRange(range)) => Some(ZeroRange { - start_line: nonnegative(range.start_line)?, - start_character: nonnegative(range.start_character)?, - end_line: nonnegative(range.end_line)?, - end_character: nonnegative(range.end_character)?, - }), - None => match occurrence.range.as_slice() { - [line, start, end] => Some(ZeroRange { - start_line: nonnegative(*line)?, - start_character: nonnegative(*start)?, - end_line: nonnegative(*line)?, - end_character: nonnegative(*end)?, - }), - [start_line, start, end_line, end] => Some(ZeroRange { - start_line: nonnegative(*start_line)?, - start_character: nonnegative(*start)?, - end_line: nonnegative(*end_line)?, - end_character: nonnegative(*end)?, - }), - _ => None, - }, - Some(_) => None, - } -} - -fn nonnegative(value: i32) -> Option { - usize::try_from(value).ok() -} - -fn line_offsets(bytes: &[u8]) -> Vec { - let mut offsets = vec![0]; - for (offset, byte) in bytes.iter().enumerate() { - if *byte == b'\n' { - offsets.push(offset.saturating_add(1)); - } - } - offsets -} - -fn resolve_range( - lines: &[usize], - bytes: &[u8], - range: ZeroRange, -) -> Option<(ByteRange, LineRange)> { - let start = lines - .get(range.start_line)? - .checked_add(range.start_character)?; - let end = lines - .get(range.end_line)? - .checked_add(range.end_character)?; - if start > end || end > bytes.len() { - return None; - } - let byte_range = ByteRange::new(u64::try_from(start).ok()?, u64::try_from(end).ok()?).ok()?; - let line_range = LineRange::new( - u32::try_from(range.start_line.checked_add(1)?).ok()?, - u32::try_from(range.end_line.checked_add(1)?).ok()?, - ) - .ok()?; - Some((byte_range, line_range)) -} - -fn scip_roles(bits: i32) -> Vec { - let mut roles = Vec::new(); - if bits & SCIP_DEFINITION != 0 { - roles.push(CodeOccurrenceRole::Definition); - } else { - roles.push(CodeOccurrenceRole::Reference); - } - for (mask, role) in [ - (SCIP_IMPORT, CodeOccurrenceRole::Import), - (SCIP_WRITE, CodeOccurrenceRole::Write), - (SCIP_READ, CodeOccurrenceRole::Read), - (SCIP_GENERATED, CodeOccurrenceRole::Generated), - (SCIP_TEST, CodeOccurrenceRole::Test), - ( - SCIP_FORWARD_DEFINITION, - CodeOccurrenceRole::ForwardDefinition, - ), - ] { - if bits & mask != 0 { - roles.push(role); - } - } - roles -} - -fn analyze_fallbacks( - sources: &[CapturedSource], - indexed_paths: &BTreeSet, - builder: &mut ArtifactBuilder, - budget: Duration, -) -> Result { - let deadline = Instant::now() + budget; - let mut used_syntax = false; - let mut used_lexical = false; - for source in sources { - if indexed_paths.contains(&source.path) { - continue; - } - if !code_index_path_is_in_scope(&source.path) { - continue; - } - if Instant::now() >= deadline { - builder.truncated = true; - builder.gap(None, CoverageGapKind::BudgetExceeded); - break; - } - builder.sources_examined = builder.sources_examined.saturating_add(1); - if std::str::from_utf8(&source.bytes).is_err() { - builder.gap( - Some(source.path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - } - #[cfg(feature = "code-rust")] - if Path::new(&source.path) - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("rs")) - { - if analyze_rust(source, builder, deadline)? { - used_syntax = true; - continue; - } - builder.gap( - Some(source.path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - } - analyze_lexical(source, builder); - used_lexical = true; - } - if used_syntax && builder.format == CodeIndexFormat::Lexical { - builder.quality = AnalysisQuality::Syntactic; - builder.format = CodeIndexFormat::Syntax; - } - Ok(match (used_syntax, used_lexical) { - (true, true) => format!("{}+texo-lexical:v2", rust_analyzer_fingerprint()), - (true, false) => rust_analyzer_fingerprint(), - (false, true) => "texo-lexical:v2".to_string(), - (false, false) => String::new(), - }) -} - -#[cfg(feature = "code-rust")] -fn analyze_rust( - source: &CapturedSource, - builder: &mut ArtifactBuilder, - deadline: Instant, -) -> Result { - use tree_sitter::Parser; - let language = tree_sitter_rust::LANGUAGE.into(); - let mut parser = Parser::new(); - parser.set_language(&language).map_err(|error| { - source_error(Path::new(&source.path), &format!("Rust grammar: {error}")) - })?; - let len = source.bytes.len(); - let mut read = |offset: usize, _| { - if offset < len { - &source.bytes[offset..] - } else { - &[] - } - }; - let mut progress = |_: &tree_sitter::ParseState| { - if Instant::now() >= deadline { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - }; - let options = tree_sitter::ParseOptions::new().progress_callback(&mut progress); - let Some(tree) = parser.parse_with_options(&mut read, None, Some(options)) else { - return Ok(false); - }; - if tree.root_node().has_error() { - builder.gap( - Some(source.path.clone()), - CoverageGapKind::AnalysisIncomplete, - ); - } - collect_rust_tags(source, builder, &language, &tree)?; - Ok(true) -} - -#[cfg(feature = "code-rust")] -fn collect_rust_tags( - source: &CapturedSource, - builder: &mut ArtifactBuilder, - language: &tree_sitter::Language, - tree: &tree_sitter::Tree, -) -> Result<(), TexoError> { - use tree_sitter::{Query, QueryCursor, StreamingIterator}; - let query = Query::new(language, tree_sitter_rust::TAGS_QUERY).map_err(|error| { - source_error( - Path::new(&source.path), - &format!("Rust tags query: {error}"), - ) - })?; - let mut cursor = QueryCursor::new(); - cursor.set_match_limit(4096); - let source_digest_hex = blake3_bytes_hex(&source.bytes); - let analyzer = rust_analyzer_fingerprint(); - let capture_names = query.capture_names(); - let mut matches = cursor.matches(&query, tree.root_node(), source.bytes.as_slice()); - while let Some(item) = matches.next() { - let role = item.captures.iter().find_map(|capture| { - let index = usize::try_from(capture.index).ok()?; - let name = capture_names.get(index)?; - if name.starts_with("definition.") { - Some((CodeOccurrenceRole::Definition, *name)) - } else if name.starts_with("reference.") { - Some((CodeOccurrenceRole::Reference, *name)) - } else { - None - } - }); - let name_capture = item.captures.iter().find(|capture| { - usize::try_from(capture.index) - .ok() - .and_then(|index| capture_names.get(index)) - .copied() - == Some("name") - }); - let (Some((role, kind)), Some(name_capture)) = (role, name_capture) else { - continue; - }; - let node = name_capture.node; - let Ok(display_name) = node.utf8_text(&source.bytes) else { - builder.gap( - Some(source.path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - }; - let symbol = format!( - "syntax rust {}#{}:{}@{}", - source.path, - kind, - display_name, - node.start_byte() - ); - let Some((context, context_byte_range, context_line_range)) = - source_context(&source.bytes, node.start_byte(), node.end_byte()) - else { - builder.gap( - Some(source.path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - }; - let occurrence = CodeOccurrence { - symbol, - display_name: display_name.to_string(), - roles: vec![role], - path: source.path.clone(), - byte_range: ByteRange::new( - u64::try_from(node.start_byte()).unwrap_or(u64::MAX), - u64::try_from(node.end_byte()).unwrap_or(u64::MAX), - ) - .map_err(|error| source_error(Path::new(&source.path), &error.to_string()))?, - line_range: LineRange::new( - u32::try_from(node.start_position().row.saturating_add(1)).unwrap_or(u32::MAX), - u32::try_from(node.end_position().row.saturating_add(1)).unwrap_or(u32::MAX), - ) - .map_err(|error| source_error(Path::new(&source.path), &error.to_string()))?, - source_digest_hex: source_digest_hex.clone(), - excerpt: display_name.to_string(), - context, - context_byte_range, - context_line_range, - analyzer_fingerprint: analyzer.clone(), - analysis_quality: AnalysisQuality::Syntactic, - }; - if !builder.push(occurrence) { - break; - } - } - if cursor.did_exceed_match_limit() { - builder.truncated = true; - builder.gap(Some(source.path.clone()), CoverageGapKind::BudgetExceeded); - } - Ok(()) -} - -#[cfg(feature = "code-rust")] -fn rust_analyzer_fingerprint() -> String { - let query_digest = blake3_bytes_hex(tree_sitter_rust::TAGS_QUERY.as_bytes()); - format!("tree-sitter:0.26.11:rust:0.24.2:tags-{query_digest}") -} - -#[cfg(not(feature = "code-rust"))] -fn rust_analyzer_fingerprint() -> String { - "tree-sitter-rust:disabled".to_string() -} - -fn analyze_lexical(source: &CapturedSource, builder: &mut ArtifactBuilder) { - let source_digest_hex = blake3_bytes_hex(&source.bytes); - let mut names = BTreeSet::new(); - let mut offset = 0; - while offset < source.bytes.len() { - if !identifier_start(source.bytes[offset]) { - offset += 1; - continue; - } - let start = offset; - offset += 1; - while offset < source.bytes.len() && identifier_continue(source.bytes[offset]) { - offset += 1; - } - let Ok(name) = std::str::from_utf8(&source.bytes[start..offset]) else { - continue; - }; - if name.len() < 3 || !names.insert(name.to_ascii_lowercase()) { - continue; - } - if names.len() > MAX_LEXICAL_OCCURRENCES_PER_SOURCE { - builder.truncated = true; - builder.gap(Some(source.path.clone()), CoverageGapKind::BudgetExceeded); - return; - } - let (start_line, end_line) = byte_line_range(&source.bytes, start, offset); - let Some((context, context_byte_range, context_line_range)) = - source_context(&source.bytes, start, offset) - else { - builder.gap( - Some(source.path.clone()), - CoverageGapKind::UnsupportedEncoding, - ); - continue; - }; - let occurrence = CodeOccurrence { - symbol: format!("lexical {}#{}@{start}", source.path, name), - display_name: name.to_string(), - roles: vec![CodeOccurrenceRole::Reference], - path: source.path.clone(), - byte_range: ByteRange { - start: u64::try_from(start).unwrap_or(u64::MAX), - end: u64::try_from(offset).unwrap_or(u64::MAX), - }, - line_range: LineRange { - start: start_line, - end: end_line, - }, - source_digest_hex: source_digest_hex.clone(), - excerpt: name.to_string(), - context, - context_byte_range, - context_line_range, - analyzer_fingerprint: "texo-lexical:v2".to_string(), - analysis_quality: AnalysisQuality::Lexical, - }; - if !builder.push(occurrence) { - return; - } - } -} - -fn code_index_path_is_in_scope(path: &str) -> bool { - let path = Path::new(path); - if path.components().any(|component| { - component.as_os_str().to_str().is_some_and(|name| { - matches!( - name.to_ascii_lowercase().as_str(), - "dist" | "build" | "node_modules" | ".next" | "coverage" - ) - }) - }) || path - .file_name() - .and_then(std::ffi::OsStr::to_str) - .is_some_and(|name| { - matches!( - name.to_ascii_lowercase().as_str(), - "pnpm-lock.yaml" | "pnpm-lock.yml" - ) - }) - { - return false; - } - // Extensionless build/config files are in the Git capture scope, so they must - // be indexable here too or a captured source would silently produce neither an - // occurrence nor a gap. - if path - .file_name() - .and_then(std::ffi::OsStr::to_str) - .is_some_and(crate::git_source::is_wellknown_source_basename) - { - return true; - } - path.extension() - .and_then(std::ffi::OsStr::to_str) - .is_some_and(|extension| { - matches!( - extension.to_ascii_lowercase().as_str(), - "rs" | "py" - | "js" - | "jsx" - | "ts" - | "tsx" - | "go" - | "java" - | "kt" - | "c" - | "h" - | "cc" - | "cpp" - | "hpp" - | "sh" - | "toml" - | "yaml" - | "yml" - | "json" - ) - }) -} - -fn identifier_start(byte: u8) -> bool { - byte == b'_' || byte.is_ascii_alphabetic() -} - -fn identifier_continue(byte: u8) -> bool { - identifier_start(byte) || byte.is_ascii_digit() -} - -fn source_context( - bytes: &[u8], - occurrence_start: usize, - occurrence_end: usize, -) -> Option<(String, ByteRange, LineRange)> { - if occurrence_start > occurrence_end || occurrence_end > bytes.len() { - return None; - } - let line_start = bytes[..occurrence_start] - .iter() - .rposition(|byte| *byte == b'\n') - .map_or(0, |position| position.saturating_add(1)); - let line_end = bytes[occurrence_end..] - .iter() - .position(|byte| *byte == b'\n') - .map_or(bytes.len(), |position| { - occurrence_end.saturating_add(position) - }); - let occurrence_len = occurrence_end.saturating_sub(occurrence_start); - if occurrence_len > MAX_EVIDENCE_EXCERPT_BYTES { - return None; - } - let available = MAX_EVIDENCE_EXCERPT_BYTES.saturating_sub(occurrence_len); - let before = available / 2; - let mut start = occurrence_start.saturating_sub(before).max(line_start); - let mut end = occurrence_end - .saturating_add(available.saturating_sub(occurrence_start.saturating_sub(start))) - .min(line_end); - if end.saturating_sub(start) < MAX_EVIDENCE_EXCERPT_BYTES { - start = end - .saturating_sub(MAX_EVIDENCE_EXCERPT_BYTES) - .max(line_start); - } - while start < occurrence_start && bytes.get(start).is_some_and(|byte| byte & 0xc0 == 0x80) { - start = start.saturating_add(1); - } - while end > occurrence_end && bytes.get(end).is_some_and(|byte| byte & 0xc0 == 0x80) { - end = end.saturating_sub(1); - } - let context = std::str::from_utf8(&bytes[start..end]).ok()?.to_string(); - let (start_line, end_line) = byte_line_range(bytes, start, end); - Some(( - context, - ByteRange { - start: u64::try_from(start).unwrap_or(u64::MAX), - end: u64::try_from(end).unwrap_or(u64::MAX), - }, - LineRange { - start: start_line, - end: end_line, - }, - )) -} - -fn byte_line_range(bytes: &[u8], start: usize, end: usize) -> (u32, u32) { - let start_line = - 1_u32.saturating_add(u32::try_from(count_newlines(&bytes[..start])).unwrap_or(u32::MAX)); - let end_line = start_line - .saturating_add(u32::try_from(count_newlines(&bytes[start..end])).unwrap_or(u32::MAX)); - (start_line, end_line) -} - -fn count_newlines(bytes: &[u8]) -> usize { - let mut count = 0_usize; - for byte in bytes { - if *byte == b'\n' { - count = count.saturating_add(1); - } - } - count -} -fn code_occurrence_order(left: &CodeOccurrence, right: &CodeOccurrence) -> std::cmp::Ordering { - left.symbol - .cmp(&right.symbol) - .then_with(|| left.path.cmp(&right.path)) - .then_with(|| left.byte_range.start.cmp(&right.byte_range.start)) - .then_with(|| left.byte_range.end.cmp(&right.byte_range.end)) - .then_with(|| left.roles.cmp(&right.roles)) -} +mod contract; -fn safe_relative_path(path: &str) -> bool { - !path.is_empty() - && !Path::new(path).is_absolute() - && Path::new(path) - .components() - .all(|component| matches!(component, Component::Normal(_))) -} +pub use contract::{CodeIndexLimits, PreparedCodeIndex}; -fn atomic_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> { - let parent = path - .parent() - .ok_or_else(|| std::io::Error::other("artifact path has no parent"))?; - std::fs::create_dir_all(parent)?; - for _attempt in 0..100 { - let counter = ARTIFACT_TMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let tmp = parent.join(format!( - ".texo-code-index-{}-{counter}.tmp", - std::process::id() - )); - let mut file = match std::fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&tmp) - { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error), - }; - let result = (|| { - file.write_all(bytes)?; - file.sync_all()?; - drop(file); - std::fs::rename(&tmp, path)?; - #[cfg(unix)] - std::fs::File::open(parent)?.sync_all()?; - Ok(()) - })(); - if result.is_err() { - let _removed = std::fs::remove_file(&tmp); - } - return result; - } - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "could not allocate a private code-index staging file", - )) -} +mod analysis; +mod artifact; +mod persistence; +mod scip_import; +mod util; -fn source_error(path: &Path, detail: &str) -> TexoError { - TexoError::Source { - path: path.display().to_string(), - detail: detail.to_string(), - } -} +pub use artifact::build; +pub use persistence::{load, persist, read_scip}; diff --git a/src/code_index/analysis.rs b/src/code_index/analysis.rs new file mode 100644 index 0000000..a867226 --- /dev/null +++ b/src/code_index/analysis.rs @@ -0,0 +1,358 @@ +use std::collections::BTreeSet; +#[cfg(feature = "code-rust")] +use std::ops::ControlFlow; +use std::path::Path; +use std::time::{Duration, Instant}; + +use crate::error::TexoError; +use crate::events::ids::blake3_bytes_hex; +use crate::git_source::CapturedSource; +use crate::knowledge::{ + AnalysisQuality, ByteRange, CodeIndexFormat, CodeOccurrence, CodeOccurrenceRole, + CoverageGapKind, LineRange, +}; + +use super::artifact::ArtifactBuilder; +#[cfg(feature = "code-rust")] +use super::util::source_error; +use super::util::{byte_line_range, source_context}; + +const MAX_LEXICAL_OCCURRENCES_PER_SOURCE: usize = 512; + +pub(super) fn analyze_fallbacks( + sources: &[CapturedSource], + indexed_paths: &BTreeSet, + builder: &mut ArtifactBuilder, + budget: Duration, +) -> Result { + let deadline = Instant::now() + budget; + let mut used_syntax = false; + let mut used_lexical = false; + for source in sources { + if indexed_paths.contains(&source.path) { + continue; + } + if !code_index_path_is_in_scope(&source.path) { + continue; + } + if Instant::now() >= deadline { + builder.truncated = true; + builder.gap(None, CoverageGapKind::BudgetExceeded); + break; + } + builder.sources_examined = builder.sources_examined.saturating_add(1); + if std::str::from_utf8(&source.bytes).is_err() { + builder.gap( + Some(source.path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + } + #[cfg(feature = "code-rust")] + if Path::new(&source.path) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("rs")) + { + if analyze_rust(source, builder, deadline)? { + used_syntax = true; + continue; + } + builder.gap( + Some(source.path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + } + analyze_lexical(source, builder); + used_lexical = true; + } + if used_syntax && builder.format == CodeIndexFormat::Lexical { + builder.quality = AnalysisQuality::Syntactic; + builder.format = CodeIndexFormat::Syntax; + } + Ok(match (used_syntax, used_lexical) { + (true, true) => format!("{}+texo-lexical:v2", rust_analyzer_fingerprint()), + (true, false) => rust_analyzer_fingerprint(), + (false, true) => "texo-lexical:v2".to_string(), + (false, false) => String::new(), + }) +} + +#[cfg(feature = "code-rust")] +fn analyze_rust( + source: &CapturedSource, + builder: &mut ArtifactBuilder, + deadline: Instant, +) -> Result { + use tree_sitter::Parser; + let language = tree_sitter_rust::LANGUAGE.into(); + let mut parser = Parser::new(); + parser.set_language(&language).map_err(|error| { + source_error(Path::new(&source.path), &format!("Rust grammar: {error}")) + })?; + let len = source.bytes.len(); + let mut read = |offset: usize, _| { + if offset < len { + &source.bytes[offset..] + } else { + &[] + } + }; + let mut progress = |_: &tree_sitter::ParseState| { + if Instant::now() >= deadline { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }; + let options = tree_sitter::ParseOptions::new().progress_callback(&mut progress); + let Some(tree) = parser.parse_with_options(&mut read, None, Some(options)) else { + return Ok(false); + }; + if tree.root_node().has_error() { + builder.gap( + Some(source.path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + } + collect_rust_tags(source, builder, &language, &tree)?; + Ok(true) +} + +#[cfg(feature = "code-rust")] +fn collect_rust_tags( + source: &CapturedSource, + builder: &mut ArtifactBuilder, + language: &tree_sitter::Language, + tree: &tree_sitter::Tree, +) -> Result<(), TexoError> { + use tree_sitter::{Query, QueryCursor, StreamingIterator}; + let query = Query::new(language, tree_sitter_rust::TAGS_QUERY).map_err(|error| { + source_error( + Path::new(&source.path), + &format!("Rust tags query: {error}"), + ) + })?; + let mut cursor = QueryCursor::new(); + cursor.set_match_limit(4096); + let source_digest_hex = blake3_bytes_hex(&source.bytes); + let analyzer = rust_analyzer_fingerprint(); + let capture_names = query.capture_names(); + let mut matches = cursor.matches(&query, tree.root_node(), source.bytes.as_slice()); + while let Some(item) = matches.next() { + let role = item.captures.iter().find_map(|capture| { + let index = usize::try_from(capture.index).ok()?; + let name = capture_names.get(index)?; + if name.starts_with("definition.") { + Some((CodeOccurrenceRole::Definition, *name)) + } else if name.starts_with("reference.") { + Some((CodeOccurrenceRole::Reference, *name)) + } else { + None + } + }); + let name_capture = item.captures.iter().find(|capture| { + usize::try_from(capture.index) + .ok() + .and_then(|index| capture_names.get(index)) + .copied() + == Some("name") + }); + let (Some((role, kind)), Some(name_capture)) = (role, name_capture) else { + continue; + }; + let node = name_capture.node; + let Ok(display_name) = node.utf8_text(&source.bytes) else { + builder.gap( + Some(source.path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + }; + let symbol = format!( + "syntax rust {}#{}:{}@{}", + source.path, + kind, + display_name, + node.start_byte() + ); + let Some((context, context_byte_range, context_line_range)) = + source_context(&source.bytes, node.start_byte(), node.end_byte()) + else { + builder.gap( + Some(source.path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + }; + let occurrence = CodeOccurrence { + symbol, + display_name: display_name.to_string(), + roles: vec![role], + path: source.path.clone(), + byte_range: ByteRange::new( + u64::try_from(node.start_byte()).unwrap_or(u64::MAX), + u64::try_from(node.end_byte()).unwrap_or(u64::MAX), + ) + .map_err(|error| source_error(Path::new(&source.path), &error.to_string()))?, + line_range: LineRange::new( + u32::try_from(node.start_position().row.saturating_add(1)).unwrap_or(u32::MAX), + u32::try_from(node.end_position().row.saturating_add(1)).unwrap_or(u32::MAX), + ) + .map_err(|error| source_error(Path::new(&source.path), &error.to_string()))?, + source_digest_hex: source_digest_hex.clone(), + excerpt: display_name.to_string(), + context, + context_byte_range, + context_line_range, + analyzer_fingerprint: analyzer.clone(), + analysis_quality: AnalysisQuality::Syntactic, + }; + if !builder.push(occurrence) { + break; + } + } + if cursor.did_exceed_match_limit() { + builder.truncated = true; + builder.gap(Some(source.path.clone()), CoverageGapKind::BudgetExceeded); + } + Ok(()) +} + +#[cfg(feature = "code-rust")] +fn rust_analyzer_fingerprint() -> String { + let query_digest = blake3_bytes_hex(tree_sitter_rust::TAGS_QUERY.as_bytes()); + format!("tree-sitter:0.26.11:rust:0.24.2:tags-{query_digest}") +} + +#[cfg(not(feature = "code-rust"))] +fn rust_analyzer_fingerprint() -> String { + "tree-sitter-rust:disabled".to_string() +} + +fn analyze_lexical(source: &CapturedSource, builder: &mut ArtifactBuilder) { + let source_digest_hex = blake3_bytes_hex(&source.bytes); + let mut names = BTreeSet::new(); + let mut offset = 0; + while offset < source.bytes.len() { + if !identifier_start(source.bytes[offset]) { + offset += 1; + continue; + } + let start = offset; + offset += 1; + while offset < source.bytes.len() && identifier_continue(source.bytes[offset]) { + offset += 1; + } + let Ok(name) = std::str::from_utf8(&source.bytes[start..offset]) else { + continue; + }; + if name.len() < 3 || !names.insert(name.to_ascii_lowercase()) { + continue; + } + if names.len() > MAX_LEXICAL_OCCURRENCES_PER_SOURCE { + builder.truncated = true; + builder.gap(Some(source.path.clone()), CoverageGapKind::BudgetExceeded); + return; + } + let (start_line, end_line) = byte_line_range(&source.bytes, start, offset); + let Some((context, context_byte_range, context_line_range)) = + source_context(&source.bytes, start, offset) + else { + builder.gap( + Some(source.path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + }; + let occurrence = CodeOccurrence { + symbol: format!("lexical {}#{}@{start}", source.path, name), + display_name: name.to_string(), + roles: vec![CodeOccurrenceRole::Reference], + path: source.path.clone(), + byte_range: ByteRange { + start: u64::try_from(start).unwrap_or(u64::MAX), + end: u64::try_from(offset).unwrap_or(u64::MAX), + }, + line_range: LineRange { + start: start_line, + end: end_line, + }, + source_digest_hex: source_digest_hex.clone(), + excerpt: name.to_string(), + context, + context_byte_range, + context_line_range, + analyzer_fingerprint: "texo-lexical:v2".to_string(), + analysis_quality: AnalysisQuality::Lexical, + }; + if !builder.push(occurrence) { + return; + } + } +} + +fn code_index_path_is_in_scope(path: &str) -> bool { + let path = Path::new(path); + if path.components().any(|component| { + component.as_os_str().to_str().is_some_and(|name| { + matches!( + name.to_ascii_lowercase().as_str(), + "dist" | "build" | "node_modules" | ".next" | "coverage" + ) + }) + }) || path + .file_name() + .and_then(std::ffi::OsStr::to_str) + .is_some_and(|name| { + matches!( + name.to_ascii_lowercase().as_str(), + "pnpm-lock.yaml" | "pnpm-lock.yml" + ) + }) + { + return false; + } + // Extensionless build/config files are in the Git capture scope, so they must + // be indexable here too or a captured source would silently produce neither an + // occurrence nor a gap. + if path + .file_name() + .and_then(std::ffi::OsStr::to_str) + .is_some_and(crate::git_source::is_wellknown_source_basename) + { + return true; + } + path.extension() + .and_then(std::ffi::OsStr::to_str) + .is_some_and(|extension| { + matches!( + extension.to_ascii_lowercase().as_str(), + "rs" | "py" + | "js" + | "jsx" + | "ts" + | "tsx" + | "go" + | "java" + | "kt" + | "c" + | "h" + | "cc" + | "cpp" + | "hpp" + | "sh" + | "toml" + | "yaml" + | "yml" + | "json" + ) + }) +} + +fn identifier_start(byte: u8) -> bool { + byte == b'_' || byte.is_ascii_alphabetic() +} + +fn identifier_continue(byte: u8) -> bool { + identifier_start(byte) || byte.is_ascii_digit() +} diff --git a/src/code_index/artifact.rs b/src/code_index/artifact.rs new file mode 100644 index 0000000..8b5afa4 --- /dev/null +++ b/src/code_index/artifact.rs @@ -0,0 +1,148 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use crate::error::TexoError; +use crate::events::ids::blake3_bytes_hex; +use crate::git_source::GitCapture; +use crate::knowledge::{ + AnalysisQuality, CodeIndexArtifact, CodeIndexFormat, CodeIndexId, CodeOccurrence, CoverageGap, + CoverageGapKind, KnowledgeCoverage, +}; + +use super::analysis::analyze_fallbacks; +use super::scip_import::import_scip; +use super::util::{code_occurrence_order, source_error}; +use super::{CodeIndexLimits, PreparedCodeIndex, ARTIFACT_SCHEMA}; + +const MAX_GAPS: usize = 256; + +/// Build a deterministic code index for one frozen Git capture. +/// +/// SCIP occurrences are preferred. Sources absent from the imported SCIP +/// index fall back to the built-in Rust grammar or bounded lexical discovery. +/// +/// # Errors +/// Returns a typed source error for malformed SCIP, invalid source ranges, or +/// artifact serialization failure. +pub fn build( + capture: &GitCapture, + scip_bytes: Option<&[u8]>, + limits: CodeIndexLimits, +) -> Result { + let source_map = capture + .sources + .iter() + .map(|source| (source.path.as_str(), source)) + .collect::>(); + let mut builder = ArtifactBuilder::new(limits, capture.coverage.clone()); + let mut indexed_paths = BTreeSet::new(); + let mut analyzer_parts = Vec::new(); + if let Some(bytes) = scip_bytes { + let before = builder.occurrences.len(); + let analyzer = import_scip(bytes, &source_map, &mut builder, &mut indexed_paths)?; + analyzer_parts.push(analyzer); + // Only claim compiler-precise coverage when SCIP actually contributed + // occurrences. An empty index, all-missing documents, or all-skipped + // ranges must not promote lexical/syntactic rows to `precise`. + if builder.occurrences.len() > before { + builder.format = CodeIndexFormat::Scip; + builder.quality = AnalysisQuality::Precise; + } + } + let fallback = analyze_fallbacks( + &capture.sources, + &indexed_paths, + &mut builder, + limits.analysis_budget, + )?; + if !fallback.is_empty() { + analyzer_parts.push(fallback); + } + if analyzer_parts.is_empty() { + analyzer_parts.push("texo-lexical:v2".to_string()); + } + let analyzer_fingerprint = analyzer_parts.join("+"); + let mut occurrences = builder.occurrences; + occurrences.sort_by(code_occurrence_order); + occurrences.dedup(); + let occurrence_material = batpak::encoding::to_bytes(&occurrences) + .map_err(|error| source_error(Path::new(".texo/cache/code-index"), &error.to_string()))?; + // Derive identity from the normalized occurrences actually persisted, not the + // raw SCIP bytes: two builds of the same snapshot+SCIP under different limits + // truncate differently, and the id must track what lands on disk so the cache + // never serves a digest-mismatched or silently truncated artifact. + let raw_digest = blake3_bytes_hex(&occurrence_material); + let index_id = CodeIndexId::derive(&format!( + "{ARTIFACT_SCHEMA}\u{1f}{}\u{1f}{raw_digest}\u{1f}{analyzer_fingerprint}", + capture.snapshot_id + )); + let coverage = KnowledgeCoverage { + analysis_quality: builder.quality, + sources_examined: builder.sources_examined, + occurrences: u64::try_from(occurrences.len()).unwrap_or(u64::MAX), + truncated: builder.truncated, + gaps: builder.gaps, + }; + let artifact = CodeIndexArtifact { + schema: ARTIFACT_SCHEMA.to_string(), + snapshot_id: capture.snapshot_id.clone(), + index_id, + format: builder.format, + analyzer_fingerprint, + occurrences, + coverage, + }; + let bytes = batpak::encoding::to_bytes(&artifact) + .map_err(|error| source_error(Path::new(".texo/cache/code-index"), &error.to_string()))?; + let artifact_digest_hex = blake3_bytes_hex(&bytes); + Ok(PreparedCodeIndex { + artifact, + artifact_digest_hex, + bytes, + }) +} + +pub(super) struct ArtifactBuilder { + pub(super) limits: CodeIndexLimits, + pub(super) occurrences: Vec, + pub(super) sources_examined: u64, + pub(super) truncated: bool, + pub(super) gaps: Vec, + pub(super) format: CodeIndexFormat, + pub(super) quality: AnalysisQuality, +} + +impl ArtifactBuilder { + pub(super) fn new(limits: CodeIndexLimits, source_coverage: KnowledgeCoverage) -> Self { + Self { + limits, + occurrences: Vec::new(), + sources_examined: 0, + truncated: source_coverage.truncated, + gaps: source_coverage.gaps, + format: CodeIndexFormat::Lexical, + quality: AnalysisQuality::Lexical, + } + } + + pub(super) fn push(&mut self, occurrence: CodeOccurrence) -> bool { + if self.occurrences.len() >= self.limits.max_occurrences { + self.truncated = true; + self.gap(None, CoverageGapKind::BudgetExceeded); + return false; + } + self.occurrences.push(occurrence); + true + } + + pub(super) fn gap(&mut self, path: Option, kind: CoverageGapKind) { + if self.gaps.len() < MAX_GAPS { + let gap = CoverageGap { path, kind }; + if !self.gaps.contains(&gap) { + self.gaps.push(gap); + } + } else { + self.truncated = true; + } + } +} diff --git a/src/code_index/contract.rs b/src/code_index/contract.rs new file mode 100644 index 0000000..7600478 --- /dev/null +++ b/src/code_index/contract.rs @@ -0,0 +1,38 @@ +use std::time::Duration; + +use crate::knowledge::CodeIndexArtifact; + +/// Bounds for one code-index build/import. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CodeIndexLimits { + /// Maximum raw SCIP bytes accepted. + pub max_scip_bytes: u64, + /// Maximum documents consumed. + pub max_documents: usize, + /// Maximum normalized occurrences retained. + pub max_occurrences: usize, + /// Global wall budget for built-in analysis. + pub analysis_budget: Duration, +} + +impl Default for CodeIndexLimits { + fn default() -> Self { + Self { + max_scip_bytes: 64 * 1024 * 1024, + max_documents: 20_000, + max_occurrences: 200_000, + analysis_budget: Duration::from_secs(30), + } + } +} + +/// A built artifact and the digest of its serialized bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreparedCodeIndex { + /// Normalized disposable artifact. + pub artifact: CodeIndexArtifact, + /// BLAKE3 digest of the exact persisted artifact bytes. + pub artifact_digest_hex: String, + /// Serialized artifact bytes. + pub bytes: Vec, +} diff --git a/src/code_index/persistence.rs b/src/code_index/persistence.rs new file mode 100644 index 0000000..e3e4686 --- /dev/null +++ b/src/code_index/persistence.rs @@ -0,0 +1,141 @@ +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::error::TexoError; +use crate::events::ids::blake3_bytes_hex; +use crate::knowledge::{CodeIndexArtifact, CodeIndexId}; + +use super::util::source_error; +use super::{PreparedCodeIndex, ARTIFACT_SCHEMA}; + +static ARTIFACT_TMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Read a workspace-local SCIP file with a hard byte bound. +/// +/// # Errors +/// Fails for paths outside the workspace, symlinks, non-regular files, and +/// files exceeding the declared bound. +pub fn read_scip(root: &Path, path: &Path, max_bytes: u64) -> Result, TexoError> { + let candidate = if path.is_absolute() { + path.to_path_buf() + } else { + root.join(path) + }; + let canonical_root = std::fs::canonicalize(root)?; + let canonical = std::fs::canonicalize(&candidate)?; + if !canonical.starts_with(&canonical_root) { + return Err(source_error(path, "SCIP path escapes the workspace")); + } + let metadata = std::fs::symlink_metadata(&candidate)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(source_error( + path, + "SCIP input must be a regular non-symlink file", + )); + } + if metadata.len() > max_bytes { + return Err(source_error( + path, + "SCIP input exceeds the configured byte limit", + )); + } + let bytes = std::fs::read(&candidate)?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) != metadata.len() { + return Err(source_error(path, "SCIP input changed while it was read")); + } + Ok(bytes) +} + +/// Persist a normalized artifact with atomic content-addressed replacement. +/// +/// # Errors +/// Returns an I/O error for staging, flush, rename, or directory sync failure. +pub fn persist(root: &Path, prepared: &PreparedCodeIndex) -> Result { + let path = artifact_path(root, &prepared.artifact.index_id); + atomic_write(&path, &prepared.bytes)?; + Ok(path) +} + +/// Load and authenticate one disposable normalized code index. +/// +/// Missing artifacts return `Ok(None)` so callers can report degraded +/// coverage. Present but malformed or digest-mismatched artifacts fail closed. +/// +/// # Errors +/// Returns a typed decode/source error when a present artifact is invalid. +pub fn load( + root: &Path, + index_id: &CodeIndexId, + expected_digest_hex: &str, +) -> Result, TexoError> { + let path = artifact_path(root, index_id); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if blake3_bytes_hex(&bytes) != expected_digest_hex { + return Err(source_error(&path, "code-index artifact digest mismatch")); + } + let artifact = batpak::encoding::from_bytes::(&bytes).map_err(|error| { + TexoError::Decode { + entity: index_id.to_string(), + detail: error.to_string(), + } + })?; + if artifact.schema != ARTIFACT_SCHEMA { + return Ok(None); + } + if artifact.index_id != *index_id { + return Err(source_error(&path, "code-index artifact identity mismatch")); + } + Ok(Some(artifact)) +} + +fn artifact_path(root: &Path, index_id: &CodeIndexId) -> PathBuf { + root.join(".texo") + .join("cache") + .join("code-index") + .join(format!("{}.bin", index_id.as_str())) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("artifact path has no parent"))?; + std::fs::create_dir_all(parent)?; + for _attempt in 0..100 { + let counter = ARTIFACT_TMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!( + ".texo-code-index-{}-{counter}.tmp", + std::process::id() + )); + let mut file = match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&tmp) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + }; + let result = (|| { + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(&tmp, path)?; + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _removed = std::fs::remove_file(&tmp); + } + return result; + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not allocate a private code-index staging file", + )) +} diff --git a/src/code_index/scip_import.rs b/src/code_index/scip_import.rs new file mode 100644 index 0000000..000c2bf --- /dev/null +++ b/src/code_index/scip_import.rs @@ -0,0 +1,281 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use protobuf::Message; + +use crate::error::TexoError; +use crate::events::ids::blake3_bytes_hex; +use crate::git_source::CapturedSource; +use crate::knowledge::{ + AnalysisQuality, ByteRange, CodeOccurrence, CodeOccurrenceRole, CoverageGapKind, LineRange, + MAX_EVIDENCE_EXCERPT_BYTES, +}; + +use super::artifact::ArtifactBuilder; +use super::util::{safe_relative_path, source_context, source_error}; + +const SCIP_DEFINITION: i32 = 0x1; +const SCIP_IMPORT: i32 = 0x2; +const SCIP_WRITE: i32 = 0x4; +const SCIP_READ: i32 = 0x8; +const SCIP_GENERATED: i32 = 0x10; +const SCIP_TEST: i32 = 0x20; +const SCIP_FORWARD_DEFINITION: i32 = 0x40; + +pub(super) fn import_scip( + bytes: &[u8], + sources: &BTreeMap<&str, &CapturedSource>, + builder: &mut ArtifactBuilder, + indexed_paths: &mut BTreeSet, +) -> Result { + let index = scip::types::Index::parse_from_bytes(bytes) + .map_err(|error| source_error(Path::new("index.scip"), &error.to_string()))?; + let analyzer = scip_analyzer_fingerprint(&index); + for document in index.documents.iter().take(builder.limits.max_documents) { + if !safe_relative_path(&document.relative_path) { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + continue; + } + builder.sources_examined = builder.sources_examined.saturating_add(1); + let Some(source) = sources.get(document.relative_path.as_str()).copied() else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::MissingObject, + ); + continue; + }; + if !position_is_utf8(document) { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + } + if std::str::from_utf8(&source.bytes).is_err() { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + } + let before = builder.occurrences.len(); + import_scip_document(document, source, &analyzer, builder); + // Only claim the path as SCIP-indexed when the document actually produced + // occurrences. A matching-but-empty document must fall through to the + // built-in fallback rather than suppress every symbol for that file. + if builder.occurrences.len() > before { + indexed_paths.insert(document.relative_path.clone()); + } + } + if index.documents.len() > builder.limits.max_documents { + builder.truncated = true; + builder.gap(None, CoverageGapKind::BudgetExceeded); + } + Ok(analyzer) +} + +fn scip_analyzer_fingerprint(index: &scip::types::Index) -> String { + let metadata = index.metadata.as_ref(); + let tool = metadata.and_then(|metadata| metadata.tool_info.as_ref()); + let name = tool.map_or("unknown", |tool| tool.name.as_str()); + let version = tool.map_or("unknown", |tool| tool.version.as_str()); + format!("scip:{name}:{version}:protocol-v0") +} + +fn position_is_utf8(document: &scip::types::Document) -> bool { + document.position_encoding.enum_value().ok() + == Some(scip::types::PositionEncoding::UTF8CodeUnitOffsetFromLineStart) +} + +fn import_scip_document( + document: &scip::types::Document, + source: &CapturedSource, + analyzer: &str, + builder: &mut ArtifactBuilder, +) { + let lines = line_offsets(&source.bytes); + let source_digest_hex = blake3_bytes_hex(&source.bytes); + for occurrence in &document.occurrences { + if occurrence.symbol.is_empty() { + continue; + } + let Some(range) = scip_range(occurrence) else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + continue; + }; + let Some((byte_range, line_range)) = resolve_range(&lines, &source.bytes, range) else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + continue; + }; + let (Ok(start), Ok(end)) = ( + usize::try_from(byte_range.start), + usize::try_from(byte_range.end), + ) else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::AnalysisIncomplete, + ); + continue; + }; + let excerpt_bytes = &source.bytes[start..end]; + let Ok(excerpt) = std::str::from_utf8(excerpt_bytes) else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + }; + if excerpt.len() > MAX_EVIDENCE_EXCERPT_BYTES { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::SourceTooLarge, + ); + continue; + } + let roles = scip_roles(occurrence.symbol_roles); + let display_name = if excerpt.is_empty() { + occurrence.symbol.clone() + } else { + excerpt.to_string() + }; + let Some((context, context_byte_range, context_line_range)) = + source_context(&source.bytes, start, end) + else { + builder.gap( + Some(document.relative_path.clone()), + CoverageGapKind::UnsupportedEncoding, + ); + continue; + }; + if !builder.push(CodeOccurrence { + symbol: occurrence.symbol.clone(), + display_name, + roles, + path: document.relative_path.clone(), + byte_range, + line_range, + source_digest_hex: source_digest_hex.clone(), + excerpt: excerpt.to_string(), + context, + context_byte_range, + context_line_range, + analyzer_fingerprint: analyzer.to_string(), + analysis_quality: AnalysisQuality::Precise, + }) { + break; + } + } +} + +#[derive(Debug, Clone, Copy)] +struct ZeroRange { + start_line: usize, + start_character: usize, + end_line: usize, + end_character: usize, +} + +fn scip_range(occurrence: &scip::types::Occurrence) -> Option { + use scip::types::occurrence::Typed_range; + match occurrence.typed_range.as_ref() { + Some(Typed_range::SingleLineRange(range)) => Some(ZeroRange { + start_line: nonnegative(range.line)?, + start_character: nonnegative(range.start_character)?, + end_line: nonnegative(range.line)?, + end_character: nonnegative(range.end_character)?, + }), + Some(Typed_range::MultiLineRange(range)) => Some(ZeroRange { + start_line: nonnegative(range.start_line)?, + start_character: nonnegative(range.start_character)?, + end_line: nonnegative(range.end_line)?, + end_character: nonnegative(range.end_character)?, + }), + None => match occurrence.range.as_slice() { + [line, start, end] => Some(ZeroRange { + start_line: nonnegative(*line)?, + start_character: nonnegative(*start)?, + end_line: nonnegative(*line)?, + end_character: nonnegative(*end)?, + }), + [start_line, start, end_line, end] => Some(ZeroRange { + start_line: nonnegative(*start_line)?, + start_character: nonnegative(*start)?, + end_line: nonnegative(*end_line)?, + end_character: nonnegative(*end)?, + }), + _ => None, + }, + Some(_) => None, + } +} + +fn nonnegative(value: i32) -> Option { + usize::try_from(value).ok() +} + +fn line_offsets(bytes: &[u8]) -> Vec { + let mut offsets = vec![0]; + for (offset, byte) in bytes.iter().enumerate() { + if *byte == b'\n' { + offsets.push(offset.saturating_add(1)); + } + } + offsets +} + +fn resolve_range( + lines: &[usize], + bytes: &[u8], + range: ZeroRange, +) -> Option<(ByteRange, LineRange)> { + let start = lines + .get(range.start_line)? + .checked_add(range.start_character)?; + let end = lines + .get(range.end_line)? + .checked_add(range.end_character)?; + if start > end || end > bytes.len() { + return None; + } + let byte_range = ByteRange::new(u64::try_from(start).ok()?, u64::try_from(end).ok()?).ok()?; + let line_range = LineRange::new( + u32::try_from(range.start_line.checked_add(1)?).ok()?, + u32::try_from(range.end_line.checked_add(1)?).ok()?, + ) + .ok()?; + Some((byte_range, line_range)) +} + +fn scip_roles(bits: i32) -> Vec { + let mut roles = Vec::new(); + if bits & SCIP_DEFINITION != 0 { + roles.push(CodeOccurrenceRole::Definition); + } else { + roles.push(CodeOccurrenceRole::Reference); + } + for (mask, role) in [ + (SCIP_IMPORT, CodeOccurrenceRole::Import), + (SCIP_WRITE, CodeOccurrenceRole::Write), + (SCIP_READ, CodeOccurrenceRole::Read), + (SCIP_GENERATED, CodeOccurrenceRole::Generated), + (SCIP_TEST, CodeOccurrenceRole::Test), + ( + SCIP_FORWARD_DEFINITION, + CodeOccurrenceRole::ForwardDefinition, + ), + ] { + if bits & mask != 0 { + roles.push(role); + } + } + roles +} diff --git a/src/code_index/util.rs b/src/code_index/util.rs new file mode 100644 index 0000000..d4dfc35 --- /dev/null +++ b/src/code_index/util.rs @@ -0,0 +1,103 @@ +use std::path::{Component, Path}; + +use crate::error::TexoError; +use crate::knowledge::{ByteRange, CodeOccurrence, LineRange, MAX_EVIDENCE_EXCERPT_BYTES}; + +pub(super) fn source_context( + bytes: &[u8], + occurrence_start: usize, + occurrence_end: usize, +) -> Option<(String, ByteRange, LineRange)> { + if occurrence_start > occurrence_end || occurrence_end > bytes.len() { + return None; + } + let line_start = bytes[..occurrence_start] + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(0, |position| position.saturating_add(1)); + let line_end = bytes[occurrence_end..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(bytes.len(), |position| { + occurrence_end.saturating_add(position) + }); + let occurrence_len = occurrence_end.saturating_sub(occurrence_start); + if occurrence_len > MAX_EVIDENCE_EXCERPT_BYTES { + return None; + } + let available = MAX_EVIDENCE_EXCERPT_BYTES.saturating_sub(occurrence_len); + let before = available / 2; + let mut start = occurrence_start.saturating_sub(before).max(line_start); + let mut end = occurrence_end + .saturating_add(available.saturating_sub(occurrence_start.saturating_sub(start))) + .min(line_end); + if end.saturating_sub(start) < MAX_EVIDENCE_EXCERPT_BYTES { + start = end + .saturating_sub(MAX_EVIDENCE_EXCERPT_BYTES) + .max(line_start); + } + while start < occurrence_start && bytes.get(start).is_some_and(|byte| byte & 0xc0 == 0x80) { + start = start.saturating_add(1); + } + while end > occurrence_end && bytes.get(end).is_some_and(|byte| byte & 0xc0 == 0x80) { + end = end.saturating_sub(1); + } + let context = std::str::from_utf8(&bytes[start..end]).ok()?.to_string(); + let (start_line, end_line) = byte_line_range(bytes, start, end); + Some(( + context, + ByteRange { + start: u64::try_from(start).unwrap_or(u64::MAX), + end: u64::try_from(end).unwrap_or(u64::MAX), + }, + LineRange { + start: start_line, + end: end_line, + }, + )) +} + +pub(super) fn byte_line_range(bytes: &[u8], start: usize, end: usize) -> (u32, u32) { + let start_line = + 1_u32.saturating_add(u32::try_from(count_newlines(&bytes[..start])).unwrap_or(u32::MAX)); + let end_line = start_line + .saturating_add(u32::try_from(count_newlines(&bytes[start..end])).unwrap_or(u32::MAX)); + (start_line, end_line) +} + +pub(super) fn count_newlines(bytes: &[u8]) -> usize { + let mut count = 0_usize; + for byte in bytes { + if *byte == b'\n' { + count = count.saturating_add(1); + } + } + count +} + +pub(super) fn code_occurrence_order( + left: &CodeOccurrence, + right: &CodeOccurrence, +) -> std::cmp::Ordering { + left.symbol + .cmp(&right.symbol) + .then_with(|| left.path.cmp(&right.path)) + .then_with(|| left.byte_range.start.cmp(&right.byte_range.start)) + .then_with(|| left.byte_range.end.cmp(&right.byte_range.end)) + .then_with(|| left.roles.cmp(&right.roles)) +} + +pub(super) fn safe_relative_path(path: &str) -> bool { + !path.is_empty() + && !Path::new(path).is_absolute() + && Path::new(path) + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +pub(super) fn source_error(path: &Path, detail: &str) -> TexoError { + TexoError::Source { + path: path.display().to_string(), + detail: detail.to_string(), + } +} diff --git a/src/config.rs b/src/config.rs index f0ca428..b73d8ad 100644 --- a/src/config.rs +++ b/src/config.rs @@ -6,9 +6,12 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::events::ids::WorkspaceId; -use crate::gateway::GatewayConfig; use crate::topology::{self, JournalEntry, ResolvedJournal}; +mod model; + +pub use model::{ConfigError, SemanticsConfig, TexoRootConfig, WorkspaceConfig}; + const DEFAULT_WORKSPACE_ID: &str = "demo"; const DEFAULT_STORE_PATH: &str = ".texo/store"; @@ -28,63 +31,6 @@ fn default_cosine_threshold() -> f32 { DEFAULT_COSINE_THRESHOLD } -/// Optional, disabled-by-default configuration for the semantic ML pipeline. -/// -/// The semantic pipeline is entirely opt-in: a workspace config without a -/// `[semantics]` table deserializes to `None`, and even when present the -/// pipeline only activates when [`SemanticsConfig::enabled`] is `true`. No ML -/// or model-runtime behavior lives here — this is configuration only. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SemanticsConfig { - /// Master switch; when `false` (the default) the pipeline is inert. - #[serde(default)] - pub enabled: bool, - /// Cosine-similarity acceptance threshold; `texo relate` uses it as the - /// cluster link threshold for candidate generation. Must sit at or below the - /// corpus's lowest same-subject similarity (default 0.65; see the module - /// source for the measured rationale). - #[serde(default = "default_cosine_threshold")] - pub cosine_threshold: f32, - /// Within-cluster pair prefilter: pairs below this cosine are never sent - /// to the judge. Defaults to the compiled 0.60; lower it (with - /// `cosine_threshold`) on recall-critical corpora — the judge remains the - /// correctness gate, so the only cost of a lower floor is judge calls. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub relate_prefilter: Option, -} - -impl Default for SemanticsConfig { - fn default() -> Self { - Self { - enabled: false, - cosine_threshold: DEFAULT_COSINE_THRESHOLD, - relate_prefilter: None, - } - } -} - -/// Resolved configuration for one `BatPak` workspace scope. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct WorkspaceConfig { - /// Workspace identifier for `BatPak` scope partitioning. - pub workspace_id: String, - /// Relative or absolute path to the `BatPak` store directory. - pub store_path: String, - /// Glob for default markdown sources. - pub docs_glob: String, - /// Optional external extractor command (newline-delimited JSON claims). - #[serde(skip_serializing_if = "Option::is_none")] - pub extractor_cmd: Option, - /// Optional, disabled-by-default semantic ML pipeline configuration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub semantics: Option, - /// Optional process-wide model gateway configuration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway: Option, -} - /// Per-workspace entry in the root config file. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -103,22 +49,9 @@ pub struct WorkspaceEntry { pub semantics: Option, } -/// Root `.texo/config.toml` with multiple workspace scopes. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TexoRootConfig { - /// Default workspace id when none is specified on the CLI. - pub default_workspace: String, - /// Named workspace configurations. - #[serde(default)] - pub workspaces: BTreeMap, - /// Optional model gateway configuration. Bootstrap never writes this field. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway: Option, -} - impl WorkspaceEntry { /// Defaults for the demo workspace. + #[must_use] pub fn demo() -> Self { let mut journals = BTreeMap::new(); journals.insert( @@ -135,6 +68,7 @@ impl WorkspaceEntry { } /// Defaults for a secondary workspace with an isolated store. + #[must_use] pub fn for_id(workspace_id: &str) -> Self { if workspace_id == DEFAULT_WORKSPACE_ID { return Self::demo(); @@ -184,6 +118,7 @@ impl WorkspaceEntry { impl TexoRootConfig { /// Default root config with only the demo workspace. + #[must_use] pub fn demo() -> Self { let mut workspaces = BTreeMap::new(); workspaces.insert(DEFAULT_WORKSPACE_ID.to_string(), WorkspaceEntry::demo()); @@ -271,6 +206,7 @@ impl TexoRootConfig { impl WorkspaceConfig { /// Default configuration for the demo workspace. + #[must_use] pub fn demo() -> Self { Self { workspace_id: DEFAULT_WORKSPACE_ID.to_string(), @@ -331,6 +267,7 @@ impl WorkspaceConfig { } /// Resolve store path relative to a workspace root. + #[must_use] pub fn store_path_buf(&self, root: &Path) -> PathBuf { let path = PathBuf::from(&self.store_path); if path.is_absolute() { @@ -347,6 +284,7 @@ impl WorkspaceConfig { /// such as `docs/**/*.md` resolves to `/docs`, while `sample_sources/**/*.md` /// resolves to `/sample_sources`. Patterns whose first component is /// already a wildcard resolve to `root` itself. + #[must_use] pub fn docs_scan_root(&self, root: &Path) -> PathBuf { let glob_path = PathBuf::from(&self.docs_glob); let mut prefix = PathBuf::new(); @@ -368,29 +306,6 @@ impl WorkspaceConfig { } } -/// Configuration-specific failures. -#[derive(Debug, thiserror::Error)] -pub enum ConfigError { - /// Filesystem error. - #[error("io: {0}")] - Io(#[from] std::io::Error), - /// TOML parse error. - #[error("parse: {0}")] - Parse(#[from] toml::de::Error), - /// TOML serialize error. - #[error("serialize: {0}")] - Serialize(#[from] toml::ser::Error), - /// Invalid workspace identifier. - #[error("invalid workspace id")] - InvalidWorkspace, - /// Unknown workspace id in root config. - #[error("unknown workspace: {0}")] - UnknownWorkspace(String), - /// Invalid journal topology or selection. - #[error("topology: {0}")] - Topology(#[from] crate::topology::TopologyError), -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/config/model.rs b/src/config/model.rs new file mode 100644 index 0000000..4bf7233 --- /dev/null +++ b/src/config/model.rs @@ -0,0 +1,96 @@ +//! Persisted configuration shapes and failures. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::{default_cosine_threshold, WorkspaceEntry, DEFAULT_COSINE_THRESHOLD}; +use crate::gateway::GatewayConfig; + +/// Optional, disabled-by-default configuration for the semantic ML pipeline. +/// +/// The semantic pipeline is entirely opt-in: a workspace config without a +/// `[semantics]` table deserializes to `None`, and even when present the +/// pipeline only activates when [`SemanticsConfig::enabled`] is `true`. No ML +/// or model-runtime behavior lives here — this is configuration only. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SemanticsConfig { + /// Master switch; when `false` (the default) the pipeline is inert. + #[serde(default)] + pub enabled: bool, + /// Cosine-similarity acceptance threshold. + #[serde(default = "default_cosine_threshold")] + pub cosine_threshold: f32, + /// Within-cluster pair prefilter. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relate_prefilter: Option, +} + +impl Default for SemanticsConfig { + fn default() -> Self { + Self { + enabled: false, + cosine_threshold: DEFAULT_COSINE_THRESHOLD, + relate_prefilter: None, + } + } +} + +/// Resolved configuration for one `BatPak` workspace scope. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfig { + /// Workspace identifier for `BatPak` scope partitioning. + pub workspace_id: String, + /// Relative or absolute path to the `BatPak` store directory. + pub store_path: String, + /// Glob for default markdown sources. + pub docs_glob: String, + /// Optional external extractor command (newline-delimited JSON claims). + #[serde(skip_serializing_if = "Option::is_none")] + pub extractor_cmd: Option, + /// Optional, disabled-by-default semantic ML pipeline configuration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub semantics: Option, + /// Optional process-wide model gateway configuration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gateway: Option, +} + +/// Root `.texo/config.toml` with multiple workspace scopes. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TexoRootConfig { + /// Default workspace id when none is specified on the CLI. + pub default_workspace: String, + /// Named workspace configurations. + #[serde(default)] + pub workspaces: BTreeMap, + /// Optional model gateway configuration. Bootstrap never writes this field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gateway: Option, +} + +/// Configuration-specific failures. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + /// Filesystem error. + #[error("io: {0}")] + Io(#[from] std::io::Error), + /// TOML parse error. + #[error("parse: {0}")] + Parse(#[from] toml::de::Error), + /// TOML serialize error. + #[error("serialize: {0}")] + Serialize(#[from] toml::ser::Error), + /// Invalid workspace identifier. + #[error("invalid workspace id")] + InvalidWorkspace, + /// Unknown workspace id in root config. + #[error("unknown workspace: {0}")] + UnknownWorkspace(String), + /// Invalid journal topology or selection. + #[error("topology: {0}")] + Topology(#[from] crate::topology::TopologyError), +} diff --git a/src/doctor.rs b/src/doctor.rs index 5c41902..edef784 100644 --- a/src/doctor.rs +++ b/src/doctor.rs @@ -8,6 +8,10 @@ use serde_json::json; use crate::config::{TexoRootConfig, WorkspaceConfig}; use crate::gateway::{ModelRole, RoleOverrides}; +mod model; + +pub use model::{DoctorCheck, DoctorReport, DoctorStatus}; + /// One diagnostic check state. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -20,49 +24,6 @@ pub enum CheckStatus { Fail, } -/// One stable diagnostic row. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct DoctorCheck { - /// Stable check identifier. - pub id: &'static str, - /// Check state. - pub status: CheckStatus, - /// Sanitized evidence. - pub detail: String, - /// Concrete repair command when one is available. - #[serde(skip_serializing_if = "Option::is_none")] - pub fix: Option, -} - -/// Aggregate doctor state. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum DoctorStatus { - /// Every required and advisory check passed. - Healthy, - /// Core is usable, but an advisory integration check needs attention. - Degraded, - /// A required config/store/verification check failed. - Broken, -} - -/// Stable machine-readable doctor report. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct DoctorReport { - /// Report schema. - pub schema: &'static str, - /// Aggregate state. - pub status: DoctorStatus, - /// Workspace selected for checks. - pub workspace_id: String, - /// Whether journal verification was requested. - pub deep: bool, - /// Whether safe managed-file repair was requested. - pub fix_requested: bool, - /// Ordered diagnostic evidence. - pub checks: Vec, -} - /// Diagnose one workspace without mutating source truth. /// /// `fix` is deliberately narrow: it only reconciles files owned by diff --git a/src/doctor/model.rs b/src/doctor/model.rs new file mode 100644 index 0000000..faf0adb --- /dev/null +++ b/src/doctor/model.rs @@ -0,0 +1,46 @@ +//! Stable diagnostic report shapes. + +use serde::Serialize; + +/// One stable diagnostic row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DoctorCheck { + /// Stable check identifier. + pub id: &'static str, + /// Check state. + pub status: super::CheckStatus, + /// Sanitized evidence. + pub detail: String, + /// Concrete repair command when one is available. + #[serde(skip_serializing_if = "Option::is_none")] + pub fix: Option, +} + +/// Aggregate doctor state. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DoctorStatus { + /// Every required and advisory check passed. + Healthy, + /// Core is usable, but an advisory integration check needs attention. + Degraded, + /// A required config/store/verification check failed. + Broken, +} + +/// Stable machine-readable doctor report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct DoctorReport { + /// Report schema. + pub schema: &'static str, + /// Aggregate state. + pub status: DoctorStatus, + /// Workspace selected for checks. + pub workspace_id: String, + /// Whether journal verification was requested. + pub deep: bool, + /// Whether safe managed-file repair was requested. + pub fix_requested: bool, + /// Ordered diagnostic evidence. + pub checks: Vec, +} diff --git a/src/error.rs b/src/error.rs index 3220668..3d7b414 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,23 +1,28 @@ //! Crate-wide texo error types. -use std::error::Error; use std::fmt; use serde::Serialize; -/// Render an error and every source in its causal chain. -#[must_use] -pub fn error_chain(error: &(dyn Error + 'static)) -> String { - let mut rendered = error.to_string(); - let mut source = error.source(); - while let Some(cause) = source { - rendered.push_str(": "); - rendered.push_str(&cause.to_string()); - source = cause.source(); +mod detail { + use std::error::Error; + + /// Render an error and every source in its causal chain. + #[must_use] + pub fn error_chain(error: &(dyn Error + 'static)) -> String { + let mut rendered = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + rendered.push_str(": "); + rendered.push_str(&cause.to_string()); + source = cause.source(); + } + rendered } - rendered } +pub use detail::error_chain; + /// Surface families that can report transport-bound errors. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SurfaceKind { @@ -161,318 +166,329 @@ impl fmt::Display for SurfaceKind { } } -/// Unified texo error with stable machine-readable codes. -#[non_exhaustive] -#[derive(Debug, thiserror::Error)] -pub enum TexoError { - /// Configuration failure. - #[error("config: {detail}")] - Config { - /// Human-readable detail. - detail: String, - /// Optional underlying configuration source error. - #[source] - source: Option>, - }, - /// Filesystem I/O error. - #[error("io: {0}")] - Io(#[from] std::io::Error), - /// JSON serialization or parsing error. - #[error("json: {0}")] - Json(#[from] serde_json::Error), - /// `BatPak` store error. - #[error("store: {0}")] - Store(#[from] batpak::store::StoreError), - /// `BatPak` coordinate construction failure. - #[error("store coordinate: {detail}")] - Coordinate { - /// Human-readable detail. - detail: String, - }, - /// `BatPak` payload registry failure. - #[error("store registry: {detail}")] - Registry { - /// Human-readable detail. - detail: String, - }, - /// Journal decode error for an entity stream. - #[error("journal decode for {entity}: {detail}")] - Decode { - /// Entity stream that could not be decoded. - entity: String, - /// Decode failure detail. - detail: String, - }, - /// Append receipt verification failure. - #[error("journal receipt for {event_id}: {reason}")] - ReceiptInvalid { - /// Event id whose append receipt failed verification. - event_id: String, - /// Rejection reason. - reason: String, - }, - /// Domain identifier parse error. - #[error("domain id: {0}")] - IdParse(#[from] crate::events::ids::IdParseError), - /// Domain status parse error. - #[error("domain status: {value}")] - StatusParse { - /// Unrecognized status string. - value: String, - }, - /// Illegal domain transition request. - #[error("domain transition {machine}: {from} -> {to}{context_suffix}", context_suffix = context.as_deref().map(|value| format!(" ({value})")).unwrap_or_default())] - Transition { - /// State machine identifier. - machine: String, - /// Source state. - from: u64, - /// Destination state. - to: u64, - /// Typed entity/state context. - context: Option, - }, - /// Required domain entity is absent. - #[error("domain missing: {entity}")] - MissingEntity { - /// Missing entity stream. - entity: String, - }, - /// Source parsing or discovery error. - #[error("source {path}: {detail}")] - Source { - /// Source path. - path: String, - /// Failure detail. - detail: String, - }, - /// Claim extraction error. - #[error("extract: {detail}")] - Extract { - /// Failure detail. - detail: String, - }, - /// Semantic backend error. - #[error("semantics {backend}: {detail}")] - Semantics { - /// Backend identifier. - backend: String, - /// Failure detail. - detail: String, - }, - /// Verification failures. - #[error("verify: {failures:?}")] - Verify { - /// Verification failure rows. - failures: Vec, - }, - /// Operation input decode or validation failure. - #[error("op input {op}: {detail}")] - OpInput { - /// Operation name. - op: String, - /// Failure detail. - detail: String, - }, - /// Operation runtime failure or denial. - #[error("op runtime {op}: {detail}")] - OpRuntime { - /// Operation name. - op: String, - /// Failure detail. - detail: String, - /// Whether the runtime denied execution. - denied: bool, - }, - /// Host composition or invocation failure. - #[error("host: {detail}")] - Host { - /// Failure detail. - detail: String, - }, - /// Surface-layer error. - #[error("surface {which}: {detail}")] - Surface { - /// Surface family. - which: SurfaceKind, - /// Failure detail. - detail: String, - }, - /// Model invocation error. - #[error("model: {detail}")] - Model { - /// Failure detail. - detail: String, - }, - /// Session handling error. - #[error("session: {detail}")] - Session { - /// Failure detail. - detail: String, - }, - /// Backup creation or verification environment failure. - #[error("backup: {detail}")] - Backup { - /// Sanitized failure detail. - detail: String, - }, - /// Snapshot-consistent read failure. - #[error("snapshot {kind}: {detail}")] - Snapshot { - /// Closed failure class. - kind: SnapshotFailureKind, - /// Sanitized failure detail. - detail: String, - }, - /// Replica bootstrap, follow, or evidence failure. - #[error("replication {kind:?}: {detail}")] - Replication { - /// Closed circuit failure class. - kind: ReplicationFailureKind, - /// What was durably changed before the failure surfaced. - committed: Committed, - /// Sanitized diagnostic detail. - detail: String, - }, -} +mod kind { + use std::error::Error; -impl TexoError { - /// Stable machine-readable error code. - #[must_use] - pub fn code(&self) -> &'static str { - match self { - Self::Config { .. } => "config", - Self::Io(_) => "io", - Self::Json(_) => "json", - Self::Store(_) => "store", - Self::Coordinate { .. } => "store.coordinate", - Self::Registry { .. } => "store.registry", - Self::Decode { .. } => "journal.decode", - Self::ReceiptInvalid { .. } => "journal.receipt", - Self::IdParse(_) => "domain.id", - Self::StatusParse { .. } => "domain.status", - Self::Transition { .. } => "domain.transition", - Self::MissingEntity { .. } => "domain.missing", - Self::Source { .. } => "source", - Self::Extract { .. } => "extract", - Self::Semantics { .. } => "semantics", - Self::Verify { .. } => "verify", - Self::OpInput { .. } => "op.input", - Self::OpRuntime { denied, .. } => { - if *denied { - "op.denied" - } else { - "op.runtime" - } - } - Self::Host { .. } => "host", - Self::Surface { which, .. } => which.code(), - Self::Model { .. } => "agent.model", - Self::Session { .. } => "agent.session", - Self::Backup { .. } => "backup", - Self::Snapshot { kind, .. } => kind.code(), - Self::Replication { kind, .. } => kind.code(), - } + use super::{ + replication_facts, Committed, FailureFacts, ReplicationFailureKind, SnapshotFailureKind, + SurfaceKind, + }; + + /// Unified texo error with stable machine-readable codes. + #[non_exhaustive] + #[derive(Debug, thiserror::Error)] + pub enum TexoError { + /// Configuration failure. + #[error("config: {detail}")] + Config { + /// Human-readable detail. + detail: String, + /// Optional underlying configuration source error. + #[source] + source: Option>, + }, + /// Filesystem I/O error. + #[error("io: {0}")] + Io(#[from] std::io::Error), + /// JSON serialization or parsing error. + #[error("json: {0}")] + Json(#[from] serde_json::Error), + /// `BatPak` store error. + #[error("store: {0}")] + Store(#[from] batpak::store::StoreError), + /// `BatPak` coordinate construction failure. + #[error("store coordinate: {detail}")] + Coordinate { + /// Human-readable detail. + detail: String, + }, + /// `BatPak` payload registry failure. + #[error("store registry: {detail}")] + Registry { + /// Human-readable detail. + detail: String, + }, + /// Journal decode error for an entity stream. + #[error("journal decode for {entity}: {detail}")] + Decode { + /// Entity stream that could not be decoded. + entity: String, + /// Decode failure detail. + detail: String, + }, + /// Append receipt verification failure. + #[error("journal receipt for {event_id}: {reason}")] + ReceiptInvalid { + /// Event id whose append receipt failed verification. + event_id: String, + /// Rejection reason. + reason: String, + }, + /// Domain identifier parse error. + #[error("domain id: {0}")] + IdParse(#[from] crate::events::ids::IdParseError), + /// Domain status parse error. + #[error("domain status: {value}")] + StatusParse { + /// Unrecognized status string. + value: String, + }, + /// Illegal domain transition request. + #[error("domain transition {machine}: {from} -> {to}{context_suffix}", context_suffix = context.as_deref().map(|value| format!(" ({value})")).unwrap_or_default())] + Transition { + /// State machine identifier. + machine: String, + /// Source state. + from: u64, + /// Destination state. + to: u64, + /// Typed entity/state context. + context: Option, + }, + /// Required domain entity is absent. + #[error("domain missing: {entity}")] + MissingEntity { + /// Missing entity stream. + entity: String, + }, + /// Source parsing or discovery error. + #[error("source {path}: {detail}")] + Source { + /// Source path. + path: String, + /// Failure detail. + detail: String, + }, + /// Claim extraction error. + #[error("extract: {detail}")] + Extract { + /// Failure detail. + detail: String, + }, + /// Semantic backend error. + #[error("semantics {backend}: {detail}")] + Semantics { + /// Backend identifier. + backend: String, + /// Failure detail. + detail: String, + }, + /// Verification failures. + #[error("verify: {failures:?}")] + Verify { + /// Verification failure rows. + failures: Vec, + }, + /// Operation input decode or validation failure. + #[error("op input {op}: {detail}")] + OpInput { + /// Operation name. + op: String, + /// Failure detail. + detail: String, + }, + /// Operation runtime failure or denial. + #[error("op runtime {op}: {detail}")] + OpRuntime { + /// Operation name. + op: String, + /// Failure detail. + detail: String, + /// Whether the runtime denied execution. + denied: bool, + }, + /// Host composition or invocation failure. + #[error("host: {detail}")] + Host { + /// Failure detail. + detail: String, + }, + /// Surface-layer error. + #[error("surface {which}: {detail}")] + Surface { + /// Surface family. + which: SurfaceKind, + /// Failure detail. + detail: String, + }, + /// Model invocation error. + #[error("model: {detail}")] + Model { + /// Failure detail. + detail: String, + }, + /// Session handling error. + #[error("session: {detail}")] + Session { + /// Failure detail. + detail: String, + }, + /// Backup creation or verification environment failure. + #[error("backup: {detail}")] + Backup { + /// Sanitized failure detail. + detail: String, + }, + /// Snapshot-consistent read failure. + #[error("snapshot {kind}: {detail}")] + Snapshot { + /// Closed failure class. + kind: SnapshotFailureKind, + /// Sanitized failure detail. + detail: String, + }, + /// Replica bootstrap, follow, or evidence failure. + #[error("replication {kind:?}: {detail}")] + Replication { + /// Closed circuit failure class. + kind: ReplicationFailureKind, + /// What was durably changed before the failure surfaced. + committed: Committed, + /// Sanitized diagnostic detail. + detail: String, + }, } - /// Recovery facts for this failure. - #[must_use] - pub fn facts(&self) -> FailureFacts { - use Committed::{No, Unknown, Yes}; - match self { - Self::Config { .. } - | Self::Json(_) - | Self::Coordinate { .. } - | Self::Registry { .. } - | Self::IdParse(_) - | Self::StatusParse { .. } - | Self::Transition { .. } - | Self::MissingEntity { .. } - | Self::Source { .. } - | Self::Extract { .. } - | Self::Verify { .. } - | Self::Backup { .. } - | Self::OpRuntime { denied: true, .. } => FailureFacts { - committed: No, - retry_safe: false, - resume: None, - }, - Self::Snapshot { kind, .. } => FailureFacts { - committed: No, - retry_safe: true, - resume: Some(match kind { - SnapshotFailureKind::InvalidToken => "fix the snapshot token and retry", - SnapshotFailureKind::Unavailable - | SnapshotFailureKind::AnchorMismatch - | SnapshotFailureKind::SourceUnavailable => { - "request the latest workspace snapshot and retry" + impl TexoError { + /// Stable machine-readable error code. + #[must_use] + pub fn code(&self) -> &'static str { + match self { + Self::Config { .. } => "config", + Self::Io(_) => "io", + Self::Json(_) => "json", + Self::Store(_) => "store", + Self::Coordinate { .. } => "store.coordinate", + Self::Registry { .. } => "store.registry", + Self::Decode { .. } => "journal.decode", + Self::ReceiptInvalid { .. } => "journal.receipt", + Self::IdParse(_) => "domain.id", + Self::StatusParse { .. } => "domain.status", + Self::Transition { .. } => "domain.transition", + Self::MissingEntity { .. } => "domain.missing", + Self::Source { .. } => "source", + Self::Extract { .. } => "extract", + Self::Semantics { .. } => "semantics", + Self::Verify { .. } => "verify", + Self::OpInput { .. } => "op.input", + Self::OpRuntime { denied, .. } => { + if *denied { + "op.denied" + } else { + "op.runtime" } - }), - }, - Self::Replication { - kind, committed, .. - } => replication_facts(*kind, *committed), - Self::OpInput { .. } => FailureFacts { - committed: No, - retry_safe: true, - resume: Some("fix the input and retry"), - }, - Self::Semantics { .. } => FailureFacts { - committed: No, - retry_safe: true, - resume: Some("run `texo relate` to resume unresolved pairs"), - }, - Self::Model { .. } => FailureFacts { - committed: Yes, - retry_safe: false, - resume: Some("user turn already recorded; re-sending duplicates it"), - }, - Self::OpRuntime { op, detail, .. } - if op == "texo.agent.chat" && detail.contains("agent.model") => - { - FailureFacts { - committed: Yes, - retry_safe: false, - resume: Some("user turn already recorded; re-sending duplicates it"), } + Self::Host { .. } => "host", + Self::Surface { which, .. } => which.code(), + Self::Model { .. } => "agent.model", + Self::Session { .. } => "agent.session", + Self::Backup { .. } => "backup", + Self::Snapshot { kind, .. } => kind.code(), + Self::Replication { kind, .. } => kind.code(), } - Self::OpRuntime { op, detail, .. } - if op == "texo.ingest.run" && detail.contains("source") => - { - FailureFacts { + } + + /// Recovery facts for this failure. + #[must_use] + pub fn facts(&self) -> FailureFacts { + use Committed::{No, Unknown, Yes}; + match self { + Self::Config { .. } + | Self::Json(_) + | Self::Coordinate { .. } + | Self::Registry { .. } + | Self::IdParse(_) + | Self::StatusParse { .. } + | Self::Transition { .. } + | Self::MissingEntity { .. } + | Self::Source { .. } + | Self::Extract { .. } + | Self::Verify { .. } + | Self::Backup { .. } + | Self::OpRuntime { denied: true, .. } => FailureFacts { committed: No, retry_safe: false, resume: None, - } - } - Self::OpRuntime { op, detail, .. } - if matches!( - op.as_str(), - "texo.claim.supersede" | "texo.conflict.resolve" - ) && detail.contains("domain.transition") => - { - FailureFacts { + }, + Self::Snapshot { kind, .. } => FailureFacts { + committed: No, + retry_safe: true, + resume: Some(match kind { + SnapshotFailureKind::InvalidToken => "fix the snapshot token and retry", + SnapshotFailureKind::Unavailable + | SnapshotFailureKind::AnchorMismatch + | SnapshotFailureKind::SourceUnavailable => { + "request the latest workspace snapshot and retry" + } + }), + }, + Self::Replication { + kind, committed, .. + } => replication_facts(*kind, *committed), + Self::OpInput { .. } => FailureFacts { committed: No, + retry_safe: true, + resume: Some("fix the input and retry"), + }, + Self::Semantics { .. } => FailureFacts { + committed: No, + retry_safe: true, + resume: Some("run `texo relate` to resume unresolved pairs"), + }, + Self::Model { .. } => FailureFacts { + committed: Yes, retry_safe: false, - resume: None, + resume: Some("user turn already recorded; re-sending duplicates it"), + }, + Self::OpRuntime { op, detail, .. } + if op == "texo.agent.chat" && detail.contains("agent.model") => + { + FailureFacts { + committed: Yes, + retry_safe: false, + resume: Some("user turn already recorded; re-sending duplicates it"), + } + } + Self::OpRuntime { op, detail, .. } + if op == "texo.ingest.run" && detail.contains("source") => + { + FailureFacts { + committed: No, + retry_safe: false, + resume: None, + } + } + Self::OpRuntime { op, detail, .. } + if matches!( + op.as_str(), + "texo.claim.supersede" | "texo.conflict.resolve" + ) && detail.contains("domain.transition") => + { + FailureFacts { + committed: No, + retry_safe: false, + resume: None, + } } + Self::Io(_) + | Self::Store(_) + | Self::Decode { .. } + | Self::ReceiptInvalid { .. } + | Self::OpRuntime { .. } + | Self::Host { .. } + | Self::Surface { .. } + | Self::Session { .. } => FailureFacts { + committed: Unknown, + retry_safe: false, + resume: Some("inspect receipts and run `texo verify` before retrying"), + }, } - Self::Io(_) - | Self::Store(_) - | Self::Decode { .. } - | Self::ReceiptInvalid { .. } - | Self::OpRuntime { .. } - | Self::Host { .. } - | Self::Surface { .. } - | Self::Session { .. } => FailureFacts { - committed: Unknown, - retry_safe: false, - resume: Some("inspect receipts and run `texo verify` before retrying"), - }, } } } +pub use kind::TexoError; + const fn replication_facts(kind: ReplicationFailureKind, committed: Committed) -> FailureFacts { let resume = match kind { ReplicationFailureKind::InvalidTopology @@ -530,13 +546,15 @@ mod tests { } } + fn assert_codes(cases: [(TexoError, &str); N]) { + for (error, expected) in cases { + assert_eq!(error.code(), expected); + } + } + #[test] - #[expect( - clippy::too_many_lines, - reason = "single exhaustive table pins every public error code" - )] - fn codes_match_the_public_error_table() { - let cases = [ + fn storage_and_journal_codes_match_the_public_table() { + assert_codes([ (config_error(), "config"), (TexoError::Io(std::io::Error::other("io")), "io"), ( @@ -576,6 +594,12 @@ mod tests { }, "journal.receipt", ), + ]); + } + + #[test] + fn domain_and_source_codes_match_the_public_table() { + assert_codes([ ( TexoError::IdParse(crate::events::ids::IdParseError::Empty), "domain.id", @@ -627,6 +651,12 @@ mod tests { }, "verify", ), + ]); + } + + #[test] + fn operation_and_surface_codes_match_the_public_table() { + assert_codes([ ( TexoError::OpInput { op: "op".to_string(), @@ -677,6 +707,12 @@ mod tests { }, "surface.cli", ), + ]); + } + + #[test] + fn agent_and_durability_codes_match_the_public_table() { + assert_codes([ ( TexoError::Model { detail: "bad".to_string(), @@ -702,11 +738,7 @@ mod tests { }, "snapshot.invalid", ), - ]; - - for (error, expected) in cases { - assert_eq!(error.code(), expected); - } + ]); } #[test] diff --git a/src/events/coordinate.rs b/src/events/coordinate.rs index 9ab7b2a..b54c788 100644 --- a/src/events/coordinate.rs +++ b/src/events/coordinate.rs @@ -3,61 +3,79 @@ use batpak::coordinate::{Coordinate, CoordinateError}; /// Coordinate scope for a workspace. +#[must_use] pub fn scope_for_workspace(workspace_id: &str) -> String { format!("workspace:{workspace_id}") } /// Entity string for a claim stream. +#[must_use] pub fn entity_for_claim(claim_id: &str) -> String { format!("claim:{claim_id}") } /// Entity string for a conflict stream. +#[must_use] pub fn entity_for_conflict(conflict_id: &str) -> String { format!("conflict:{conflict_id}") } /// Entity string for a source stream. +#[must_use] pub fn entity_for_source(source_id: &str) -> String { format!("source:{source_id}") } /// Entity string for the onboarding projection stream. +#[must_use] pub fn entity_for_onboarding_projection() -> String { "projection:onboarding".to_string() } /// Entity string for workspace metadata. +#[must_use] pub fn entity_for_workspace_meta(workspace_id: &str) -> String { format!("workspace-meta:{workspace_id}") } /// Entity string for a session stream. +#[must_use] pub fn entity_for_session(session_id: &str) -> String { format!("session:{session_id}") } /// Entity string for one provider-neutral logical relation pair. +#[must_use] pub fn entity_for_relation_pair(pair_id: &str) -> String { format!("relation:{pair_id}") } +/// Entity string for one workspace relation campaign. +#[must_use] +pub fn entity_for_relation_campaign(workspace_id: &str) -> String { + format!("relation-campaign:{workspace_id}") +} + /// Entity string for one frozen source snapshot. +#[must_use] pub fn entity_for_source_snapshot(snapshot_id: &str) -> String { format!("source-snapshot:{snapshot_id}") } /// Entity string for one evidence occurrence. +#[must_use] pub fn entity_for_evidence(occurrence_id: &str) -> String { format!("evidence:{occurrence_id}") } /// Entity string for one disposable code-index registration. +#[must_use] pub fn entity_for_code_index(index_id: &str) -> String { format!("code-index:{index_id}") } /// Entity string for one directional frozen-snapshot comparison. +#[must_use] pub fn entity_for_source_relation(left_snapshot_id: &str, right_snapshot_id: &str) -> String { format!("source-relation:{left_snapshot_id}:{right_snapshot_id}") } @@ -162,6 +180,17 @@ pub fn coordinate_for_relation_pair( ) } +/// Build a workspace relation-campaign coordinate. +/// +/// # Errors +/// Returns [`CoordinateError`] if the generated coordinate is invalid. +pub fn coordinate_for_relation_campaign(workspace_id: &str) -> Result { + Coordinate::new( + entity_for_relation_campaign(workspace_id), + scope_for_workspace(workspace_id), + ) +} + /// Build a frozen source-snapshot coordinate. /// /// # Errors @@ -220,6 +249,7 @@ pub fn coordinate_for_source_relation( } /// Deterministically map a session id to a non-zero `BatPak` lane. +#[must_use] pub fn session_lane(session_id: &str) -> u32 { let hash = blake3::hash(session_id.as_bytes()); let bytes = hash.as_bytes(); @@ -239,6 +269,10 @@ mod tests { assert_eq!(entity_for_onboarding_projection(), "projection:onboarding"); assert_eq!(entity_for_workspace_meta("demo"), "workspace-meta:demo"); assert_eq!(entity_for_session("s1"), "session:s1"); + assert_eq!( + entity_for_relation_campaign("demo"), + "relation-campaign:demo" + ); assert_eq!( entity_for_source_snapshot("snapshot_abc"), "source-snapshot:snapshot_abc" @@ -263,6 +297,7 @@ mod tests { coordinate_for_onboarding_projection("demo").expect("projection coordinate"), coordinate_for_workspace_meta("demo").expect("workspace metadata coordinate"), coordinate_for_session("demo", "session_abc").expect("session coordinate"), + coordinate_for_relation_campaign("demo").expect("campaign coordinate"), coordinate_for_source_snapshot("demo", "snapshot_abc") .expect("source snapshot coordinate"), coordinate_for_evidence("demo", "evidence_abc").expect("evidence coordinate"), diff --git a/src/events/ids.rs b/src/events/ids.rs index fe4a8ae..7d3bd1a 100644 --- a/src/events/ids.rs +++ b/src/events/ids.rs @@ -122,11 +122,13 @@ impl WorkspaceId { } /// Borrow the inner string slice. + #[must_use] pub fn as_str(&self) -> &str { &self.0 } /// `BatPak` scope string: `workspace:{id}`. + #[must_use] pub fn scope(&self) -> String { format!("workspace:{}", self.0) } @@ -176,6 +178,7 @@ pub fn source_id_from_hash(body_hash_hex: &str) -> Result ConflictId { let (left, right) = if a.as_str() <= b.as_str() { (a.as_str(), b.as_str()) @@ -215,11 +219,13 @@ pub fn relation_pair_id( } /// BLAKE3 hex digest for app-level content hashing (distinct from `BatPak` event hashes). +#[must_use] pub fn blake3_hash_hex(input: &str) -> String { blake3::hash(input.as_bytes()).to_hex().to_string() } /// BLAKE3 hex digest of raw bytes. +#[must_use] pub fn blake3_bytes_hex(input: &[u8]) -> String { blake3::hash(input).to_hex().to_string() } diff --git a/src/events/inventory.rs b/src/events/inventory.rs new file mode 100644 index 0000000..72eb854 --- /dev/null +++ b/src/events/inventory.rs @@ -0,0 +1,112 @@ +//! Canonical inventory of Texo event payload schemas. + +use batpak::event::{EventKind, EventPayload}; + +use crate::events::payloads::{ + ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, CodeIndexRecordedV1, + ConflictOpenedV2, ConflictResolvedV2, EvidenceOccurrenceRecordedV1, + EvidenceReconciliationAcceptedV1, OnboardingCompiledV2, RelationCampaignCheckpointV1, + RelationDeferredV1, RelationJudgedV1, ReplicaBatchMaterializedV1, SessionTurnV1, + SourceObservedV2, SourceSnapshotRecordedV1, SourceSnapshotRelationV1, WorkspaceInitializedV2, +}; + +/// One typed event payload binding exposed through `hostbat`. +pub(crate) struct EventSchema { + /// `BatPak` event kind owned by the payload type. + pub(crate) kind: EventKind, + /// Stable schema identifier mounted into the host module. + pub(crate) schema_ref: &'static str, + rust_type: fn() -> &'static str, +} + +impl EventSchema { + /// Fully qualified Rust type name for host diagnostics. + pub(crate) fn rust_type(&self) -> &'static str { + (self.rust_type)() + } +} + +const fn event_schema( + schema_ref: &'static str, + rust_type: fn() -> &'static str, +) -> EventSchema { + EventSchema { + kind: T::KIND, + schema_ref, + rust_type, + } +} + +fn rust_type() -> &'static str { + std::any::type_name::() +} + +/// Complete, stable event payload surface mounted by Texo's host module. +pub(crate) const EVENT_SCHEMAS: &[EventSchema] = &[ + event_schema::("texo.event.claim-recorded.v2", rust_type::), + event_schema::( + "texo.event.claim-superseded.v2", + rust_type::, + ), + event_schema::( + "texo.event.conflict-opened.v2", + rust_type::, + ), + event_schema::( + "texo.event.conflict-resolved.v2", + rust_type::, + ), + event_schema::( + "texo.event.source-observed.v2", + rust_type::, + ), + event_schema::( + "texo.event.onboarding-compiled.v2", + rust_type::, + ), + event_schema::( + "texo.event.workspace-initialized.v2", + rust_type::, + ), + event_schema::( + "texo.event.relation-judged.v1", + rust_type::, + ), + event_schema::( + "texo.event.relation-deferred.v1", + rust_type::, + ), + event_schema::( + "texo.event.source-snapshot-recorded.v1", + rust_type::, + ), + event_schema::( + "texo.event.evidence-occurrence-recorded.v1", + rust_type::, + ), + event_schema::( + "texo.event.evidence-reconciliation-accepted.v1", + rust_type::, + ), + event_schema::( + "texo.event.claim-evidence-linked.v1", + rust_type::, + ), + event_schema::( + "texo.event.code-index-recorded.v1", + rust_type::, + ), + event_schema::( + "texo.event.source-snapshot-relation.v1", + rust_type::, + ), + event_schema::("texo.event.session-turn.v1", rust_type::), + event_schema::( + "texo.event.replica-batch-materialized.v1", + rust_type::, + ), + event_schema::( + "texo.event.relation-campaign-checkpoint.v1", + rust_type::, + ), +]; diff --git a/src/events/machines.rs b/src/events/machines.rs index e632180..22752be 100644 --- a/src/events/machines.rs +++ b/src/events/machines.rs @@ -8,11 +8,6 @@ use crate::events::payloads::{ }; mod claim_markers { - #![expect( - missing_docs, - reason = "batpak::define_state_machine! does not accept docs for generated markers" - )] - batpak::define_state_machine!( claim_seal, ClaimPhase { @@ -24,11 +19,6 @@ mod claim_markers { } mod conflict_markers { - #![expect( - missing_docs, - reason = "batpak::define_state_machine! does not accept docs for generated markers" - )] - batpak::define_state_machine!( conflict_seal, ConflictPhase { @@ -41,23 +31,29 @@ mod conflict_markers { } /// Claim phase marker trait. -pub use claim_markers::ClaimPhase; +pub trait ClaimPhase: claim_markers::ClaimPhase {} + +impl ClaimPhase for T {} + /// Current claim phase marker. -pub use claim_markers::Current; +pub type Current = claim_markers::Current; /// Superseded claim phase marker. -pub use claim_markers::Superseded; +pub type Superseded = claim_markers::Superseded; /// Unrecorded claim phase marker. -pub use claim_markers::Unrecorded; +pub type Unrecorded = claim_markers::Unrecorded; /// Conflict phase marker trait. -pub use conflict_markers::ConflictPhase; +pub trait ConflictPhase: conflict_markers::ConflictPhase {} + +impl ConflictPhase for T {} + /// Ignored conflict phase marker. -pub use conflict_markers::Ignored; +pub type Ignored = conflict_markers::Ignored; /// Open conflict phase marker. -pub use conflict_markers::Open; +pub type Open = conflict_markers::Open; /// Resolved conflict phase marker. -pub use conflict_markers::Resolved; +pub type Resolved = conflict_markers::Resolved; /// Unopened conflict phase marker. -pub use conflict_markers::Unopened; +pub type Unopened = conflict_markers::Unopened; /// Claim state machine identifier. pub const CLAIM_MACHINE: &str = "texo.claim.v2"; @@ -95,6 +91,7 @@ pub struct TransitionCauseV1 { } /// Build a transition record for a legal domain edge. +#[must_use] pub fn transition_record( machine: &str, entity: &str, @@ -121,6 +118,7 @@ pub fn transition_record( } /// Deterministically derive a transition id. +#[must_use] pub fn transition_id( machine: &str, entity: &str, @@ -148,11 +146,13 @@ pub fn transition_id( } /// Construct the only exported claim-record transition. +#[must_use] pub fn record_claim(payload: ClaimRecordedV2) -> Transition { Transition::from_payload(payload) } /// Construct the only exported claim-supersede transition. +#[must_use] pub fn supersede_claim( payload: ClaimSupersededV2, ) -> Transition { @@ -160,11 +160,13 @@ pub fn supersede_claim( } /// Construct the only exported conflict-open transition. +#[must_use] pub fn open_conflict(payload: ConflictOpenedV2) -> Transition { Transition::from_payload(payload) } /// Construct the only exported conflict-resolve transition. +#[must_use] pub fn resolve_conflict( payload: ConflictResolvedV2, ) -> Transition { @@ -172,6 +174,7 @@ pub fn resolve_conflict( } /// Construct the only exported conflict-ignore transition. +#[must_use] pub fn ignore_conflict( payload: ConflictResolvedV2, ) -> Transition { diff --git a/src/events/mod.rs b/src/events/mod.rs index a27905e..e8e73c5 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -2,5 +2,6 @@ pub mod coordinate; pub mod ids; +pub(crate) mod inventory; pub mod machines; pub mod payloads; diff --git a/src/events/payloads.rs b/src/events/payloads.rs index e813711..6f66c7e 100644 --- a/src/events/payloads.rs +++ b/src/events/payloads.rs @@ -15,6 +15,12 @@ use crate::knowledge::{ }; use crate::relate::settlement::{RelationFailureClass, SettledRelation}; +mod campaign; +mod session; + +pub use campaign::RelationCampaignCheckpointV1; +pub use session::SessionTurnV1; + /// A source document observation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, batpak::EventPayload)] #[batpak(category = 0xE, type_id = 1, version = 2)] @@ -167,24 +173,6 @@ pub struct WorkspaceInitializedV2 { pub created_at_ms: u64, } -/// One turn in a session transcript. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, batpak::EventPayload)] -#[batpak(category = 0xE, type_id = 8, version = 1)] -pub struct SessionTurnV1 { - /// Stable session identifier. - pub session_id: String, - /// Workspace scope identifier. - pub workspace_id: String, - /// Speaker label, either `user` or `assistant`. - pub speaker: String, - /// Turn text. - pub text: String, - /// Monotonic turn number within the session. - pub turn_no: u32, - /// Observation wall-clock time in milliseconds. - pub observed_at_ms: u64, -} - /// A completed semantic judgment for one logical relation pair. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, batpak::EventPayload)] #[batpak(category = 0xE, type_id = 9, version = 1)] diff --git a/src/events/payloads/campaign.rs b/src/events/payloads/campaign.rs new file mode 100644 index 0000000..e9c2acd --- /dev/null +++ b/src/events/payloads/campaign.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; + +use crate::events::ids::WorkspaceId; +use crate::relate::settlement::CampaignPhase; + +/// Durable progress proof for one workspace relation campaign. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, batpak::EventPayload)] +#[batpak(category = 0xE, type_id = 18, version = 1)] +pub struct RelationCampaignCheckpointV1 { + /// Workspace scope identifier. + pub workspace_id: WorkspaceId, + /// Claim basis evaluated by this page. + pub evaluated_basis_digest_hex: String, + /// Claim basis resulting after any complete-page authority publication. + pub result_basis_digest_hex: String, + /// Candidate-discovery policy identity, excluding credentials. + pub candidate_policy_digest_hex: String, + /// Closed partial-or-complete campaign state. + pub phase: CampaignPhase, + /// Observation wall-clock time in milliseconds; never an ordering input. + pub observed_at_ms: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn partial_checkpoint_round_trips_canonically() { + let checkpoint = RelationCampaignCheckpointV1 { + workspace_id: WorkspaceId::try_from("workspace").expect("valid workspace"), + evaluated_basis_digest_hex: "a".repeat(64), + result_basis_digest_hex: "a".repeat(64), + candidate_policy_digest_hex: "b".repeat(64), + phase: CampaignPhase::Partial { + next_candidate_cursor: 17, + }, + observed_at_ms: 23, + }; + let encoded = batpak::canonical::to_bytes(&checkpoint).expect("checkpoint encodes"); + let decoded = batpak::canonical::from_bytes(&encoded).expect("checkpoint decodes"); + assert_eq!(checkpoint, decoded); + } +} diff --git a/src/events/payloads/session.rs b/src/events/payloads/session.rs new file mode 100644 index 0000000..c4711e9 --- /dev/null +++ b/src/events/payloads/session.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +/// One turn in a session transcript. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, batpak::EventPayload)] +#[batpak(category = 0xE, type_id = 8, version = 1)] +pub struct SessionTurnV1 { + /// Stable session identifier. + pub session_id: String, + /// Workspace scope identifier. + pub workspace_id: String, + /// Speaker label, either `user` or `assistant`. + pub speaker: String, + /// Turn text. + pub text: String, + /// Monotonic turn number within the session. + pub turn_no: u32, + /// Observation wall-clock time in milliseconds. + pub observed_at_ms: u64, +} diff --git a/src/extract/cache.rs b/src/extract/cache.rs index ac9fada..bfe6939 100644 --- a/src/extract/cache.rs +++ b/src/extract/cache.rs @@ -144,6 +144,21 @@ impl CachingRelater { fn path_for(&self, key: &str) -> PathBuf { self.dir.join(format!("{key}.json")) } + + /// Remove the cached verdict for one ordered pair so the next call must + /// consult the wrapped relater. Missing entries are already fresh. + /// + /// # Errors + /// + /// Returns an I/O error when an existing cache entry cannot be removed. + pub fn evict(&self, older: &str, newer: &str) -> std::io::Result { + let path = self.path_for(&self.cache_key(older, newer)); + match std::fs::remove_file(path) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } + } } impl ClaimRelater for CachingRelater { @@ -306,6 +321,28 @@ mod tests { let _removed = std::fs::remove_dir_all(&dir); } + #[test] + fn explicit_eviction_forces_one_fresh_relation_call() { + let dir = tmp_dir().join("relate-evict"); + let _removed = std::fs::remove_dir_all(&dir); + let relater = CachingRelater::new( + CountingRelater { + calls: Cell::new(0), + relation: ClaimRelation::Conflict, + fingerprint: "fp-evict".to_string(), + }, + dir.clone(), + ); + relater.relate("old", "new").expect("cold relation"); + relater.relate("old", "new").expect("cached relation"); + assert_eq!(relater.inner.calls.get(), 1); + assert!(relater.evict("old", "new").expect("evict existing")); + assert!(!relater.evict("old", "new").expect("missing is inert")); + relater.relate("old", "new").expect("fresh relation"); + assert_eq!(relater.inner.calls.get(), 2); + let _removed = std::fs::remove_dir_all(&dir); + } + #[test] fn corrupt_entry_is_recomputed() { let dir = tmp_dir().join("corrupt"); diff --git a/src/extract/faithfulness.rs b/src/extract/faithfulness.rs index 8a87e60..24dffd2 100644 --- a/src/extract/faithfulness.rs +++ b/src/extract/faithfulness.rs @@ -58,6 +58,7 @@ fn content_tokens(text: &str) -> HashSet { /// Returns the recall and the pass/fail verdict. A claim with no content tokens is /// never grounded (recall `0`), so an empty or punctuation-only proposal cannot /// slip through. +#[must_use] pub fn assess_faithfulness( claim_text: &str, source_text: &str, diff --git a/src/extract/heuristics.rs b/src/extract/heuristics.rs index 6da94b2..470c44d 100644 --- a/src/extract/heuristics.rs +++ b/src/extract/heuristics.rs @@ -3,6 +3,7 @@ use super::word_match::contains_any; /// Returns true when a markdown line should become a claim in v1. +#[must_use] pub fn is_claim_line(line: &str) -> bool { let trimmed = line.trim(); if trimmed.is_empty() { diff --git a/src/extract/hints.rs b/src/extract/hints.rs index e6273ce..ac1ea29 100644 --- a/src/extract/hints.rs +++ b/src/extract/hints.rs @@ -44,12 +44,14 @@ const KEYWORDS: &[(&str, u32)] = &[ ]; /// Derive hints from a raw markdown line. +#[must_use] pub fn hints_from_line(line: &str) -> Option { hints_from_line_normalized(line, &normalize_line(line)) } /// Like [`hints_from_line`] for callers that already normalized the line — /// the extract hot loop otherwise normalizes every candidate line twice. +#[must_use] pub fn hints_from_line_normalized(line: &str, normalized: &str) -> Option { if !super::heuristics::is_claim_line(line) { return None; diff --git a/src/extract/markdown.rs b/src/extract/markdown.rs index e2ac7d3..1a13378 100644 --- a/src/extract/markdown.rs +++ b/src/extract/markdown.rs @@ -163,10 +163,6 @@ struct HeadingFrame { /// tables, and blockquotes (and all their contents) are excluded. Empty or /// degenerate input yields an empty vector. #[must_use] -#[expect( - clippy::wildcard_enum_match_arm, - reason = "pulldown-cmark Event is a foreign enum that grows variants; only the matched events matter here" -)] pub fn segment_candidates(source: &str) -> Vec { let mut options = Options::empty(); options.insert(Options::ENABLE_TABLES); @@ -191,59 +187,68 @@ pub fn segment_candidates(source: &str) -> Vec { let mut open_block: Option<(usize, usize)> = None; for (event, range) in Parser::new_ext(source, options).into_offset_iter() { - match event { - Event::Start(Tag::Heading { level, .. }) => { - suppress_depth += 1; - capturing_heading = Some((level, String::new())); - } - Event::End(TagEnd::Heading(_)) => { - if let Some((level, text)) = capturing_heading.take() { - while headings.last().is_some_and(|frame| frame.level >= level) { - headings.pop(); - } - headings.push(HeadingFrame { - level, - text: text.trim().to_string(), - }); + if let Event::Start(Tag::Heading { level, .. }) = &event { + suppress_depth += 1; + capturing_heading = Some((*level, String::new())); + continue; + } + if matches!(&event, Event::End(TagEnd::Heading(_))) { + if let Some((level, text)) = capturing_heading.take() { + while headings.last().is_some_and(|frame| frame.level >= level) { + headings.pop(); } - suppress_depth = suppress_depth.saturating_sub(1); + headings.push(HeadingFrame { + level, + text: text.trim().to_string(), + }); } + suppress_depth = suppress_depth.saturating_sub(1); + continue; + } + if matches!( + &event, Event::Start( Tag::CodeBlock(_) - | Tag::Table(_) - | Tag::BlockQuote(_) - | Tag::HtmlBlock - | Tag::MetadataBlock(_), - ) => { - suppress_depth += 1; - } + | Tag::Table(_) + | Tag::BlockQuote(_) + | Tag::HtmlBlock + | Tag::MetadataBlock(_) + ) + ) { + suppress_depth += 1; + continue; + } + if matches!( + &event, Event::End( TagEnd::CodeBlock - | TagEnd::Table - | TagEnd::BlockQuote(_) - | TagEnd::HtmlBlock - | TagEnd::MetadataBlock(_), - ) => { - suppress_depth = suppress_depth.saturating_sub(1); - } - Event::Start(Tag::Paragraph | Tag::Item) => { - if suppress_depth == 0 && open_block.is_none() { - open_block = Some((range.start, range.end)); - } + | TagEnd::Table + | TagEnd::BlockQuote(_) + | TagEnd::HtmlBlock + | TagEnd::MetadataBlock(_) + ) + ) { + suppress_depth = suppress_depth.saturating_sub(1); + continue; + } + if matches!(&event, Event::Start(Tag::Paragraph | Tag::Item)) { + if suppress_depth == 0 && open_block.is_none() { + open_block = Some((range.start, range.end)); } - Event::End(TagEnd::Paragraph | TagEnd::Item) => { - if suppress_depth == 0 { - if let Some((start, end)) = open_block.take() { - push_span(&mut spans, source, start, end, &headings); - } + continue; + } + if matches!(&event, Event::End(TagEnd::Paragraph | TagEnd::Item)) { + if suppress_depth == 0 { + if let Some((start, end)) = open_block.take() { + push_span(&mut spans, source, start, end, &headings); } } - Event::Text(text) | Event::Code(text) => { - if let Some((_, buf)) = capturing_heading.as_mut() { - buf.push_str(&text); - } + continue; + } + if let Event::Text(text) | Event::Code(text) = &event { + if let Some((_, buf)) = capturing_heading.as_mut() { + buf.push_str(text); } - _ => {} } } diff --git a/src/extract/mod.rs b/src/extract/mod.rs index b65d406..c2bfc77 100644 --- a/src/extract/mod.rs +++ b/src/extract/mod.rs @@ -3,11 +3,15 @@ /// Default heuristic confidence in parts per million. pub const DEFAULT_CONFIDENCE_PPM: u32 = 500_000; -pub mod faithfulness; pub mod cache; +/// Claim grounding and faithfulness assessment. +pub mod faithfulness; pub mod heuristics; +/// Extracted claim hint surface. pub mod hints; pub mod llm; +/// Markdown discovery and segmentation. pub mod markdown; +/// Claim text normalization. pub mod normalize; pub mod word_match; diff --git a/src/extract/normalize.rs b/src/extract/normalize.rs index f17ee31..2165058 100644 --- a/src/extract/normalize.rs +++ b/src/extract/normalize.rs @@ -1,6 +1,7 @@ //! Line normalization for claim extraction. /// Normalize a line for claim identity and comparison. +#[must_use] pub fn normalize_line(line: &str) -> String { let lower = line.trim().to_ascii_lowercase(); let mut out = String::with_capacity(lower.len()); diff --git a/src/extract/word_match.rs b/src/extract/word_match.rs index e081af8..b6d27ab 100644 --- a/src/extract/word_match.rs +++ b/src/extract/word_match.rs @@ -17,6 +17,7 @@ fn words(text: &str) -> impl Iterator { /// /// `needle` is matched case-insensitively against complete word tokens, so it /// will not match when it is only a substring of a larger word. +#[must_use] pub fn contains_word(text: &str, needle: &str) -> bool { let needle = needle.trim(); if needle.is_empty() { @@ -27,6 +28,7 @@ pub fn contains_word(text: &str, needle: &str) -> bool { /// Returns true when `phrase` (one or more whitespace-separated words) occurs as /// a consecutive run of whole words in `text`. +#[must_use] pub fn contains_phrase(text: &str, phrase: &str) -> bool { let needle: Vec<&str> = phrase.split_whitespace().collect(); if needle.is_empty() { @@ -48,6 +50,7 @@ pub fn contains_phrase(text: &str, phrase: &str) -> bool { } /// Returns true when any of `needles` occurs as a whole word or phrase in `text`. +#[must_use] pub fn contains_any(text: &str, needles: &[&str]) -> bool { let haystack = words(text).collect::>(); needles.iter().any(|needle| { diff --git a/src/gateway.rs b/src/gateway.rs index 43835fe..6bdfefa 100644 --- a/src/gateway.rs +++ b/src/gateway.rs @@ -4,6 +4,10 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +mod model; + +pub use model::{GatewayConfig, GatewayEnvironment}; + /// Built-in provider identifier and profile. pub const DEFAULT_PROVIDER_ID: &str = "openrouter"; /// Neutral environment variable for the OpenAI-compatible base URL. @@ -141,25 +145,6 @@ pub struct RoleConfig { pub response_format: ResponseFormatPolicy, } -/// Optional `[gateway]` configuration. Absence means built-in defaults. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct GatewayConfig { - /// Named provider profiles. - pub providers: BTreeMap, - /// Optional embedding-role override. - pub embed: Option, - /// Optional proposal-role override. - pub propose: Option, - /// Optional relation-role override. - pub relate: Option, - /// Optional chat-role override. - pub chat: Option, - /// Global semantic relate wall-clock budget. - #[serde(default = "default_relate_budget_secs")] - pub relate_budget_secs: u64, -} - impl Default for GatewayConfig { fn default() -> Self { Self { @@ -195,17 +180,6 @@ pub struct RoleOverrides { pub model: Option, } -/// Already-read neutral environment values used by the pure resolver. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct GatewayEnvironment { - /// `TEXO_LLM_BASE_URL` value. - pub base_url: Option, - /// Value read from the selected provider profile's key environment name. - pub api_key: Option, - /// Role-specific `TEXO_LLM_*_MODEL` value. - pub model: Option, -} - /// Resolved provider and role configuration, including the secret key. #[derive(Debug, Clone, PartialEq)] pub struct ResolvedRole { diff --git a/src/gateway/model.rs b/src/gateway/model.rs new file mode 100644 index 0000000..3448b66 --- /dev/null +++ b/src/gateway/model.rs @@ -0,0 +1,37 @@ +//! Gateway configuration shapes whose public names remain stable by re-export. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::{ProviderProfile, RoleConfig}; + +/// Optional `[gateway]` configuration. Absence means built-in defaults. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct GatewayConfig { + /// Named provider profiles. + pub providers: BTreeMap, + /// Optional embedding-role override. + pub embed: Option, + /// Optional proposal-role override. + pub propose: Option, + /// Optional relation-role override. + pub relate: Option, + /// Optional chat-role override. + pub chat: Option, + /// Global semantic relate wall-clock budget. + #[serde(default = "super::default_relate_budget_secs")] + pub relate_budget_secs: u64, +} + +/// Already-read neutral environment values used by the pure resolver. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GatewayEnvironment { + /// `TEXO_LLM_BASE_URL` value. + pub base_url: Option, + /// Value read from the selected provider profile's key environment name. + pub api_key: Option, + /// Role-specific `TEXO_LLM_*_MODEL` value. + pub model: Option, +} diff --git a/src/host/mod.rs b/src/host/mod.rs index 91f75ff..223a28f 100644 --- a/src/host/mod.rs +++ b/src/host/mod.rs @@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex}; use batpak::coordinate::Coordinate; use batpak::store::{Open, ReadOnly, Store, StoreConfig}; -use serde::{Deserialize, Serialize}; use syncbat::{RuntimeError, StoreOperationStatusSink, StoreReceiptSink}; use crate::claims::workspace::WorkspaceCache; @@ -20,51 +19,12 @@ use crate::ops::backend::TexoEffectBackend; use crate::ops::env::{self, OpEnv}; use crate::topology::{JournalRole, ResolvedJournal}; -/// Cross-request checkout slot for the warm workspace projection. -pub type SharedWorkspaceCache = Arc>>; - -/// Deterministic fingerprints exposed by the composed texo host. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HostFingerprints { - /// Digest of the texo operation module catalog. - pub module_digest: String, - /// Digest of the runnable host composition. - pub host_fingerprint: String, - /// Digest of the client-visible operation interface. - pub interface_fingerprint: String, -} - -/// One public operation in the mounted `hostbat` interface. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HostOperationView { - /// Stable operation name. - pub name: String, - /// Stable effect class spelling. - pub effect: String, - /// Stable receipt schema reference. - pub receipt_kind: String, -} +mod model; -/// Client-visible projection of the actual mounted `hostbat` composition. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HostInterface { - /// Interface schema identifier. - pub schema: String, - /// Texo binary version. - pub version: String, - /// Content identities produced by `hostbat`. - pub fingerprints: HostFingerprints, - /// Canonically ordered mounted operations. - pub operations: Vec, -} +pub use model::{HostFingerprints, HostInterface, HostOperationView, TexoHost}; -/// Runnable Texo host over a content-identified `hostbat` composition. -pub struct TexoHost { - host: hostbat::Host, - env: Rc, - interface: HostInterface, - shared_cache: Option, -} +/// Cross-request checkout slot for the warm workspace projection. +pub type SharedWorkspaceCache = Arc>>; /// Open the configured workspace store. /// @@ -302,7 +262,7 @@ impl TexoHost { &RoleOverrides::default(), config.gateway.as_ref(), ); - if grants_model_capability(Some(model_role.api_key)) { + if grants_model_capability(Some(model_role.api_key.as_str())) { builder = builder.grant_capability("texo.cap.model"); } let host = builder.build().map_err(build_error)?; @@ -446,12 +406,8 @@ impl Drop for TexoHost { /// Pure capability gate for the optional model capability. #[must_use] -#[expect( - clippy::needless_pass_by_value, - reason = "WO-2a requires the pure capability helper to take Option" -)] -pub fn grants_model_capability(value: Option) -> bool { - value.as_deref().is_some_and(|key| !key.trim().is_empty()) +pub fn grants_model_capability(value: Option<&str>) -> bool { + value.is_some_and(|key| !key.trim().is_empty()) } fn load_or_default_config( @@ -654,9 +610,9 @@ mod tests { #[test] fn model_capability_gate_is_non_empty_key_only() { assert!(!grants_model_capability(None)); - assert!(!grants_model_capability(Some(String::new()))); - assert!(!grants_model_capability(Some(" ".to_string()))); - assert!(grants_model_capability(Some("sk-test".to_string()))); + assert!(!grants_model_capability(Some(""))); + assert!(!grants_model_capability(Some(" "))); + assert!(grants_model_capability(Some("sk-test"))); } #[test] diff --git a/src/host/model.rs b/src/host/model.rs new file mode 100644 index 0000000..e69e40d --- /dev/null +++ b/src/host/model.rs @@ -0,0 +1,51 @@ +//! Host identity and runtime shapes re-exported by the host facade. + +use std::rc::Rc; + +use serde::{Deserialize, Serialize}; + +use super::SharedWorkspaceCache; +use crate::ops::env::OpEnv; + +/// Deterministic fingerprints exposed by the composed texo host. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HostFingerprints { + /// Digest of the texo operation module catalog. + pub module_digest: String, + /// Digest of the runnable host composition. + pub host_fingerprint: String, + /// Digest of the client-visible operation interface. + pub interface_fingerprint: String, +} + +/// One public operation in the mounted `hostbat` interface. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HostOperationView { + /// Stable operation name. + pub name: String, + /// Stable effect class spelling. + pub effect: String, + /// Stable receipt schema reference. + pub receipt_kind: String, +} + +/// Client-visible projection of the actual mounted `hostbat` composition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HostInterface { + /// Interface schema identifier. + pub schema: String, + /// Texo binary version. + pub version: String, + /// Content identities produced by `hostbat`. + pub fingerprints: HostFingerprints, + /// Canonically ordered mounted operations. + pub operations: Vec, +} + +/// Runnable Texo host over a content-identified `hostbat` composition. +pub struct TexoHost { + pub(super) host: hostbat::Host, + pub(super) env: Rc, + pub(super) interface: HostInterface, + pub(super) shared_cache: Option, +} diff --git a/src/host/module.rs b/src/host/module.rs index 5ecffcb..08c2757 100644 --- a/src/host/module.rs +++ b/src/host/module.rs @@ -2,7 +2,6 @@ use std::collections::BTreeSet; -use batpak::event::EventPayload; use hostbat::{ DiagnosticRustType, GoldenVector, GuardDescriptor, HostModule, HostModuleBuilder, SchemaDescriptor, SchemaId, SchemaRole, SchemaVersion, @@ -10,13 +9,7 @@ use hostbat::{ use syncbat::{AdmissionDecision, OperationDescriptor}; use crate::error::TexoError; -use crate::events::payloads::{ - ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, CodeIndexRecordedV1, - ConflictOpenedV2, ConflictResolvedV2, EvidenceOccurrenceRecordedV1, - EvidenceReconciliationAcceptedV1, OnboardingCompiledV2, RelationDeferredV1, RelationJudgedV1, - ReplicaBatchMaterializedV1, SessionTurnV1, SourceObservedV2, SourceSnapshotRecordedV1, - SourceSnapshotRelationV1, WorkspaceInitializedV2, -}; +use crate::events::inventory::EVENT_SCHEMAS; use crate::topology::JournalRole; /// Stable module id folded into the `hostbat` composition fingerprint. @@ -84,56 +77,18 @@ fn declare_operation_schemas( } fn declare_event_payloads(mut builder: HostModuleBuilder) -> Result { - macro_rules! bind { - ($payload:ty, $schema:literal) => {{ - builder = builder - .schema(schema_descriptor( - $schema, - SchemaRole::EventPayload, - Some(std::any::type_name::<$payload>()), - )?) - .map_err(host_error)?; - builder = builder - .bind_event_payload(<$payload as EventPayload>::KIND, $schema) - .map_err(host_error)?; - }}; + for event in EVENT_SCHEMAS { + builder = builder + .schema(schema_descriptor( + event.schema_ref, + SchemaRole::EventPayload, + Some(event.rust_type()), + )?) + .map_err(host_error)?; + builder = builder + .bind_event_payload(event.kind, event.schema_ref) + .map_err(host_error)?; } - - bind!(ClaimRecordedV2, "texo.event.claim-recorded.v2"); - bind!(ClaimSupersededV2, "texo.event.claim-superseded.v2"); - bind!(ConflictOpenedV2, "texo.event.conflict-opened.v2"); - bind!(ConflictResolvedV2, "texo.event.conflict-resolved.v2"); - bind!(SourceObservedV2, "texo.event.source-observed.v2"); - bind!(OnboardingCompiledV2, "texo.event.onboarding-compiled.v2"); - bind!( - WorkspaceInitializedV2, - "texo.event.workspace-initialized.v2" - ); - bind!(RelationJudgedV1, "texo.event.relation-judged.v1"); - bind!(RelationDeferredV1, "texo.event.relation-deferred.v1"); - bind!( - SourceSnapshotRecordedV1, - "texo.event.source-snapshot-recorded.v1" - ); - bind!( - EvidenceOccurrenceRecordedV1, - "texo.event.evidence-occurrence-recorded.v1" - ); - bind!( - EvidenceReconciliationAcceptedV1, - "texo.event.evidence-reconciliation-accepted.v1" - ); - bind!(ClaimEvidenceLinkedV1, "texo.event.claim-evidence-linked.v1"); - bind!(CodeIndexRecordedV1, "texo.event.code-index-recorded.v1"); - bind!( - SourceSnapshotRelationV1, - "texo.event.source-snapshot-relation.v1" - ); - bind!(SessionTurnV1, "texo.event.session-turn.v1"); - bind!( - ReplicaBatchMaterializedV1, - "texo.event.replica-batch-materialized.v1" - ); Ok(builder) } @@ -175,7 +130,10 @@ fn host_error(error: impl std::fmt::Display) -> TexoError { #[cfg(test)] mod tests { + use batpak::event::EventPayload; + use super::*; + use crate::events::payloads::ReplicaBatchMaterializedV1; #[test] fn module_seals_the_complete_catalog_and_event_surface() { @@ -185,7 +143,10 @@ mod tests { module.manifest().operations().count(), crate::ops::catalog().len() ); - assert_eq!(module.manifest().event_payload_bindings().count(), 17); + assert_eq!( + module.manifest().event_payload_bindings().count(), + EVENT_SCHEMAS.len() + ); } #[test] diff --git a/src/install.rs b/src/install.rs index 79423c7..d4ad345 100644 --- a/src/install.rs +++ b/src/install.rs @@ -1,34 +1,36 @@ //! Idempotent workspace appliance installation. use std::collections::BTreeSet; -use std::io::Write as _; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; use clap::ValueEnum; -use indexmap::IndexMap; use serde::Serialize; -use serde_json::{json, Value}; use crate::error::TexoError; use crate::topology::{JournalEntry, JournalRole, ReplicaMode}; +mod adapter; +mod entry; +mod filesystem; +mod model; + +use adapter::{ + canonical_manifest, client_path, managed_created_paths, managed_journal_id, + managed_server_entry, managed_workspace_id, merge_client_adapter, preflight_remove_client, + remove_client, resolve_clients, upsert_agent_guide, AGENT_MARKER_END, AGENT_MARKER_START, +}; +use filesystem::{ + ensure_safe_managed_path, remove_managed_file, remove_marked_block, with_newline, write_managed, +}; + +pub use entry::install; +pub use model::{InstallChange, InstallReport}; + /// Canonical client-neutral MCP manifest. pub const MCP_MANIFEST_PATH: &str = ".texo/mcp.json"; /// Root agent guidance file updated through one managed marker block. pub const AGENT_GUIDE_PATH: &str = "AGENTS.md"; -const CLAUDE_MCP_PATH: &str = ".mcp.json"; -const CURSOR_MCP_PATH: &str = ".cursor/mcp.json"; -const CODEX_CONFIG_PATH: &str = ".codex/config.toml"; -const CODEX_MARKER_START: &str = "# texo:install:codex:start"; -const CODEX_MARKER_END: &str = "# texo:install:codex:end"; -const AGENT_MARKER_START: &str = ""; -const AGENT_MARKER_END: &str = ""; -static INSTALL_TMP_COUNTER: AtomicU64 = AtomicU64::new(0); - -type OrderedJsonObject = IndexMap>; - /// Agent client adapter target. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, ValueEnum)] #[serde(rename_all = "lowercase")] @@ -59,34 +61,6 @@ pub enum ChangeAction { Removed, } -/// One install/uninstall path result. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct InstallChange { - /// Workspace-relative path. - pub path: String, - /// Action applied or planned. - pub action: ChangeAction, -} - -/// Machine-readable appliance installation report. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct InstallReport { - /// Report schema. - pub schema: &'static str, - /// Workspace root. - pub root: String, - /// Workspace id installed. - pub workspace_id: String, - /// Whether this was a write-free preview. - pub dry_run: bool, - /// Selected concrete clients. - pub clients: Vec, - /// Physical read journal selected for each client adapter. - pub routes: Vec, - /// Ordered path changes. - pub changes: Vec, -} - /// One agent adapter's scale-out journal route. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ClientJournalRoute { @@ -111,31 +85,48 @@ pub struct UninstallReport { pub changes: Vec, } -/// Install the lightweight Texo appliance. +/// Install the appliance with every client pinned to one physical journal. /// /// # Errors -/// Returns an error when existing client configuration is malformed, already -/// owns a conflicting `texo` entry, or a managed write fails. -pub fn install( +/// Returns the same safe-merge failures as [`install`]. +pub fn install_for_journal( root: &Path, workspace_id: &str, + journal_id: Option<&str>, requested: &[ClientTarget], dry_run: bool, ) -> Result { - install_for_journal(root, workspace_id, None, requested, dry_run) + let plan = prepare_install_plan(root, workspace_id, journal_id, requested)?; + let changes = apply_install_plan(root, workspace_id, journal_id, dry_run, &plan)?; + Ok(InstallReport { + schema: "texo.install.v2", + root: root.display().to_string(), + workspace_id: workspace_id.to_string(), + dry_run, + clients: plan.clients, + routes: plan.routes, + changes, + }) } -/// Install the appliance with every client pinned to one physical journal. -/// -/// # Errors -/// Returns the same safe-merge failures as [`install`]. -pub fn install_for_journal( +struct InstallPlan { + clients: Vec, + created_paths: BTreeSet, + config_path: PathBuf, + config_existed: bool, + decision: crate::surfaces::bootstrap::BootstrapDecision, + root_config: crate::config::TexoRootConfig, + routes: Vec, + topology_changed: bool, + previous_server: Option, +} + +fn prepare_install_plan( root: &Path, workspace_id: &str, journal_id: Option<&str>, requested: &[ClientTarget], - dry_run: bool, -) -> Result { +) -> Result { let clients = resolve_clients(root, requested); let created_paths = prepare_install_paths(root, &clients)?; let config_path = root.join(".texo/config.toml"); @@ -165,38 +156,77 @@ pub fn install_for_journal( )?; } upsert_agent_guide(root, workspace_id, true)?; + Ok(InstallPlan { + clients, + created_paths, + config_path, + config_existed, + decision, + root_config, + routes, + topology_changed, + previous_server, + }) +} +fn apply_install_plan( + root: &Path, + workspace_id: &str, + journal_id: Option<&str>, + dry_run: bool, + plan: &InstallPlan, +) -> Result, TexoError> { let mut changes = Vec::new(); + changes.push(apply_install_topology(root, workspace_id, dry_run, plan)?); + append_managed_surface_changes(&mut changes, root, workspace_id, journal_id, dry_run, plan)?; + Ok(changes) +} + +fn apply_install_topology( + root: &Path, + workspace_id: &str, + dry_run: bool, + plan: &InstallPlan, +) -> Result { if !dry_run { - crate::surfaces::bootstrap::ensure_workspace(root, workspace_id, &decision)?; - if topology_changed { - root_config - .save(&config_path) + crate::surfaces::bootstrap::ensure_workspace(root, workspace_id, &plan.decision)?; + if plan.topology_changed { + plan.root_config + .save(&plan.config_path) .map_err(|error| TexoError::Config { detail: error.to_string(), source: Some(Box::new(error)), })?; } - for route in &routes { + for route in &plan.routes { let _report = crate::replication::refresh_reader(root, Some(workspace_id), &route.journal_id)?; } } - changes.push(InstallChange { + Ok(InstallChange { path: ".texo/config.toml".to_string(), - action: if !config_existed { + action: if !plan.config_existed { ChangeAction::Created - } else if topology_changed { + } else if plan.topology_changed { ChangeAction::Updated } else { ChangeAction::Unchanged }, - }); + }) +} +fn append_managed_surface_changes( + changes: &mut Vec, + root: &Path, + workspace_id: &str, + journal_id: Option<&str>, + dry_run: bool, + plan: &InstallPlan, +) -> Result<(), TexoError> { let canonical = serde_json::to_vec_pretty(&canonical_manifest( workspace_id, journal_id, - &created_paths, + &plan.created_paths, ))?; changes.push(write_managed( root, @@ -215,13 +245,13 @@ pub fn install_for_journal( dry_run, )?); - for client in &clients { + for client in &plan.clients { let Some(change) = merge_client_adapter( root, workspace_id, *client, - route_for(&routes, *client), - previous_server.as_ref(), + route_for(&plan.routes, *client), + plan.previous_server.as_ref(), dry_run, )? else { @@ -230,15 +260,7 @@ pub fn install_for_journal( changes.push(change); } changes.push(upsert_agent_guide(root, workspace_id, dry_run)?); - Ok(InstallReport { - schema: "texo.install.v2", - root: root.display().to_string(), - workspace_id: workspace_id.to_string(), - dry_run, - clients, - routes, - changes, - }) + Ok(()) } fn prepare_install_paths( @@ -268,37 +290,6 @@ fn prepare_install_paths( Ok(created_paths) } -fn merge_client_adapter( - root: &Path, - workspace_id: &str, - client: ClientTarget, - journal_id: Option<&str>, - previous_managed: Option<&Value>, - dry_run: bool, -) -> Result, TexoError> { - let change = match client { - ClientTarget::Claude => merge_json_adapter( - root, - CLAUDE_MCP_PATH, - workspace_id, - journal_id, - previous_managed, - dry_run, - )?, - ClientTarget::Cursor => merge_json_adapter( - root, - CURSOR_MCP_PATH, - workspace_id, - journal_id, - previous_managed, - dry_run, - )?, - ClientTarget::Codex => merge_codex_adapter(root, workspace_id, journal_id, dry_run)?, - ClientTarget::Auto | ClientTarget::All => return Ok(None), - }; - Ok(Some(change)) -} - fn plan_client_routes( config: &mut crate::config::TexoRootConfig, workspace_id: &str, @@ -384,21 +375,6 @@ fn route_for(routes: &[ClientJournalRoute], client: ClientTarget) -> Option<&str .map(|route| route.journal_id.as_str()) } -fn managed_server_entry(root: &Path) -> Result, TexoError> { - let path = root.join(MCP_MANIFEST_PATH); - if !path.exists() { - return Ok(None); - } - let manifest = serde_json::from_slice::(&std::fs::read(path)?)?; - if manifest.get("schema").and_then(Value::as_str) != Some("texo.mcp-install.v1") { - return Err(config_error( - MCP_MANIFEST_PATH, - "existing manifest is not owned by this installer", - )); - } - Ok(manifest.get("server").cloned()) -} - /// Remove only Texo-managed appliance entries, never journals or config. /// /// # Errors @@ -503,631 +479,6 @@ pub fn uninstall( }) } -fn resolve_clients(root: &Path, requested: &[ClientTarget]) -> Vec { - let requested = if requested.is_empty() { - &[ClientTarget::Auto][..] - } else { - requested - }; - let mut selected = BTreeSet::new(); - for target in requested { - match target { - ClientTarget::All => { - selected.extend([ - ClientTarget::Codex, - ClientTarget::Claude, - ClientTarget::Cursor, - ]); - } - ClientTarget::Auto => { - if root.join(".codex").exists() { - selected.insert(ClientTarget::Codex); - } - if root.join(CLAUDE_MCP_PATH).exists() || root.join(".claude").exists() { - selected.insert(ClientTarget::Claude); - } - if root.join(".cursor").exists() { - selected.insert(ClientTarget::Cursor); - } - } - concrete @ (ClientTarget::Codex | ClientTarget::Claude | ClientTarget::Cursor) => { - selected.insert(*concrete); - } - } - } - selected.into_iter().collect() -} - -fn canonical_manifest( - workspace_id: &str, - journal_id: Option<&str>, - created_paths: &BTreeSet, -) -> Value { - json!({ - "schema": "texo.mcp-install.v1", - "server": server_entry(workspace_id, journal_id), - "created_paths": created_paths - }) -} - -fn server_entry(workspace_id: &str, journal_id: Option<&str>) -> Value { - let mut args = vec![ - "--root".to_string(), - ".".to_string(), - "--workspace".to_string(), - workspace_id.to_string(), - ]; - if let Some(journal_id) = journal_id { - args.push("--journal".to_string()); - args.push(journal_id.to_string()); - } - args.push("mcp".to_string()); - json!({ - "command": "texo", - "args": args, - "env": {} - }) -} - -fn merge_json_adapter( - root: &Path, - relative: &str, - workspace_id: &str, - journal_id: Option<&str>, - previous_managed: Option<&Value>, - dry_run: bool, -) -> Result { - ensure_safe_managed_path(root, relative)?; - let path = root.join(relative); - let existed = path.exists(); - let (mut document, mut servers) = read_ordered_adapter(&path, existed, relative)?; - let wanted = server_entry(workspace_id, journal_id); - if let Some(existing) = servers.get("texo") { - let existing = serde_json::from_str::(existing.get())?; - if existing != wanted && previous_managed != Some(&existing) { - return Err(config_error( - relative, - "existing mcpServers.texo is not managed by this installer", - )); - } - } - servers.insert("texo".to_string(), raw_json(&wanted)?); - document.insert("mcpServers".to_string(), raw_json(&servers)?); - let bytes = with_newline(serde_json::to_vec_pretty(&document)?); - let action = classify_bytes(&path, &bytes)?; - if !dry_run && action != ChangeAction::Unchanged { - atomic_write(&path, &bytes)?; - } - Ok(InstallChange { - path: relative.to_string(), - action: if !existed && action == ChangeAction::Updated { - ChangeAction::Created - } else { - action - }, - }) -} - -fn merge_codex_adapter( - root: &Path, - workspace_id: &str, - journal_id: Option<&str>, - dry_run: bool, -) -> Result { - ensure_safe_managed_path(root, CODEX_CONFIG_PATH)?; - let path = root.join(CODEX_CONFIG_PATH); - let existing = read_optional_string(&path)?; - let (without, had_marker) = - strip_marked_block(&existing, CODEX_MARKER_START, CODEX_MARKER_END)?; - if !without.trim().is_empty() { - let parsed = without - .parse::() - .map_err(|error| TexoError::Config { - detail: format!("{CODEX_CONFIG_PATH}: {error}"), - source: Some(Box::new(error)), - })?; - if parsed - .get("mcp_servers") - .and_then(|value| value.get("texo")) - .is_some() - { - return Err(config_error( - CODEX_CONFIG_PATH, - "existing mcp_servers.texo is outside the managed block", - )); - } - } - let journal_args = journal_id.map_or_else(String::new, |journal_id| { - format!(", \"--journal\", \"{}\"", escape_toml(journal_id)) - }); - let args = format!( - "[\"--root\", \".\", \"--workspace\", \"{}\"{journal_args}, \"mcp\"]", - escape_toml(workspace_id) - ); - let block = format!( - "{CODEX_MARKER_START}\n[mcp_servers.texo]\ncommand = \"texo\"\nargs = {args}\n{CODEX_MARKER_END}" - ); - let updated = append_block(&without, &block); - let bytes = updated.into_bytes(); - let action = classify_bytes(&path, &bytes)?; - if !dry_run && action != ChangeAction::Unchanged { - atomic_write(&path, &bytes)?; - } - Ok(InstallChange { - path: CODEX_CONFIG_PATH.to_string(), - action: if !path.exists() && !had_marker { - ChangeAction::Created - } else { - action - }, - }) -} - -fn upsert_agent_guide( - root: &Path, - workspace_id: &str, - dry_run: bool, -) -> Result { - ensure_safe_managed_path(root, AGENT_GUIDE_PATH)?; - let path = root.join(AGENT_GUIDE_PATH); - let existing = read_optional_string(&path)?; - let (without, _) = strip_marked_block(&existing, AGENT_MARKER_START, AGENT_MARKER_END)?; - let block = format!( - "{AGENT_MARKER_START}\n## Texo agent context\n\nWorkspace: `{workspace_id}`. Start with the `get_agent_context` MCP tool before answering from project knowledge. Reuse its snapshot token with `search_knowledge`, `explain_knowledge`, and `triangulate` so one investigation stays on one frontier. Inspect coverage before treating absence as evidence. Absence of a relation verdict never means unrelated. Texo MCP tools are local and read-only.\n{AGENT_MARKER_END}" - ); - let bytes = append_block(&without, &block).into_bytes(); - let action = classify_bytes(&path, &bytes)?; - if !dry_run && action != ChangeAction::Unchanged { - atomic_write(&path, &bytes)?; - } - Ok(InstallChange { - path: AGENT_GUIDE_PATH.to_string(), - action: if path.exists() { - action - } else { - ChangeAction::Created - }, - }) -} - -fn client_path(client: ClientTarget) -> Option<&'static str> { - match client { - ClientTarget::Codex => Some(CODEX_CONFIG_PATH), - ClientTarget::Claude => Some(CLAUDE_MCP_PATH), - ClientTarget::Cursor => Some(CURSOR_MCP_PATH), - ClientTarget::Auto | ClientTarget::All => None, - } -} - -fn managed_manifest(root: &Path) -> Result, TexoError> { - ensure_safe_managed_path(root, MCP_MANIFEST_PATH)?; - let path = root.join(MCP_MANIFEST_PATH); - if !path.exists() { - return Ok(None); - } - let document = serde_json::from_slice::(&std::fs::read(path)?)?; - if document.get("schema").and_then(Value::as_str) != Some("texo.mcp-install.v1") { - return Err(config_error( - MCP_MANIFEST_PATH, - "file is not managed by this installer", - )); - } - Ok(Some(document)) -} - -fn managed_created_paths(root: &Path) -> Result, TexoError> { - let Some(document) = managed_manifest(root)? else { - return Ok(BTreeSet::new()); - }; - document.get("created_paths").map_or_else( - || Ok(BTreeSet::new()), - |paths| serde_json::from_value(paths.clone()).map_err(TexoError::Json), - ) -} - -fn managed_workspace_id(root: &Path) -> Result, TexoError> { - let Some(document) = managed_manifest(root)? else { - return Ok(None); - }; - // Recover the workspace from the `--workspace ` flag pair, not a fixed - // positional index. A managed manifest with a valid schema but an - // unexpected args shape must fail closed rather than silently rewrite the - // manifest to point future MCP clients at the `demo` workspace. - let args = document - .get("server") - .and_then(|server| server.get("args")) - .and_then(Value::as_array) - .ok_or_else(|| config_error(MCP_MANIFEST_PATH, "managed manifest has no server args"))?; - let workspace = args.windows(2).find_map(|pair| { - if pair[0].as_str() == Some("--workspace") { - pair[1].as_str() - } else { - None - } - }); - match workspace { - Some(id) => Ok(Some(id.to_string())), - None => Err(config_error( - MCP_MANIFEST_PATH, - "managed manifest args carry no recoverable --workspace id; refusing to rewrite", - )), - } -} - -fn managed_journal_id(root: &Path) -> Result, TexoError> { - let Some(document) = managed_manifest(root)? else { - return Ok(None); - }; - let args = document - .get("server") - .and_then(|server| server.get("args")) - .and_then(Value::as_array) - .ok_or_else(|| config_error(MCP_MANIFEST_PATH, "managed manifest has no server args"))?; - Ok(args.windows(2).find_map(|pair| { - if pair[0].as_str() == Some("--journal") { - pair[1].as_str().map(str::to_string) - } else { - None - } - })) -} - -fn preflight_remove_client( - root: &Path, - client: ClientTarget, - created_paths: &BTreeSet, -) -> Result<(), TexoError> { - let _change = remove_client(root, client, created_paths, true)?; - Ok(()) -} - -fn remove_client( - root: &Path, - client: ClientTarget, - created_paths: &BTreeSet, - dry_run: bool, -) -> Result, TexoError> { - let Some(relative) = client_path(client) else { - return Ok(None); - }; - let remove_empty = created_paths.contains(relative); - match client { - ClientTarget::Claude | ClientTarget::Cursor => { - remove_json_adapter(root, relative, remove_empty, dry_run) - } - ClientTarget::Codex => remove_marked_block( - root, - relative, - CODEX_MARKER_START, - CODEX_MARKER_END, - remove_empty, - dry_run, - ), - ClientTarget::Auto | ClientTarget::All => Ok(None), - } -} - -fn remove_json_adapter( - root: &Path, - relative: &str, - remove_empty: bool, - dry_run: bool, -) -> Result, TexoError> { - ensure_safe_managed_path(root, relative)?; - let path = root.join(relative); - if !path.exists() { - return Ok(None); - } - let (mut document, mut servers) = read_ordered_adapter(&path, true, relative)?; - let Some(existing) = servers.get("texo") else { - return Ok(None); - }; - if !is_managed_server_entry(&serde_json::from_str::(existing.get())?) { - return Err(config_error( - relative, - "mcpServers.texo is not managed by this installer", - )); - } - servers.shift_remove("texo"); - document.insert("mcpServers".to_string(), raw_json(&servers)?); - if !dry_run { - if remove_empty && json_adapter_is_empty(&document, &servers) { - std::fs::remove_file(&path)?; - } else { - atomic_write(&path, &with_newline(serde_json::to_vec_pretty(&document)?))?; - } - } - Ok(Some(InstallChange { - path: relative.to_string(), - action: ChangeAction::Removed, - })) -} - -fn read_ordered_adapter( - path: &Path, - existed: bool, - relative: &str, -) -> Result<(OrderedJsonObject, OrderedJsonObject), TexoError> { - let document = if existed { - serde_json::from_slice::(&std::fs::read(path)?)? - } else { - OrderedJsonObject::new() - }; - let servers = document.get("mcpServers").map_or_else( - || Ok(OrderedJsonObject::new()), - |raw| { - serde_json::from_str::(raw.get()).map_err(|error| { - TexoError::Config { - detail: format!("{relative}: mcpServers must be an object"), - source: Some(Box::new(error)), - } - }) - }, - )?; - Ok((document, servers)) -} - -fn raw_json(value: &T) -> Result, TexoError> { - serde_json::value::RawValue::from_string(serde_json::to_string(value)?).map_err(TexoError::Json) -} - -fn json_adapter_is_empty(document: &OrderedJsonObject, servers: &OrderedJsonObject) -> bool { - document.len() == 1 && servers.is_empty() -} - -fn remove_managed_file( - root: &Path, - relative: &str, - schema: &str, - dry_run: bool, -) -> Result { - ensure_safe_managed_path(root, relative)?; - let path = root.join(relative); - let action = if path.is_file() { - let document = serde_json::from_slice::(&std::fs::read(&path)?)?; - if document.get("schema").and_then(Value::as_str) != Some(schema) { - return Err(config_error( - relative, - "file is not managed by this installer", - )); - } - if !dry_run { - std::fs::remove_file(path)?; - } - ChangeAction::Removed - } else { - ChangeAction::Unchanged - }; - Ok(InstallChange { - path: relative.to_string(), - action, - }) -} - -fn is_managed_server_entry(value: &Value) -> bool { - value.get("command").and_then(Value::as_str) == Some("texo") - && value - .get("args") - .and_then(Value::as_array) - .is_some_and(|args| { - args.first().and_then(Value::as_str) == Some("--root") - && args.get(1).and_then(Value::as_str) == Some(".") - && args.get(2).and_then(Value::as_str) == Some("--workspace") - && args.get(3).and_then(Value::as_str).is_some() - && args.last().and_then(Value::as_str) == Some("mcp") - && (args.len() == 5 - || (args.len() == 7 - && args.get(4).and_then(Value::as_str) == Some("--journal") - && args.get(5).and_then(Value::as_str).is_some())) - }) -} - -fn remove_marked_block( - root: &Path, - relative: &str, - start: &str, - end: &str, - remove_empty: bool, - dry_run: bool, -) -> Result, TexoError> { - ensure_safe_managed_path(root, relative)?; - let path = root.join(relative); - let existing = read_optional_string(&path)?; - let (without, had_marker) = strip_marked_block(&existing, start, end)?; - if !had_marker { - return Ok(None); - } - if !dry_run { - if remove_empty && without.trim().is_empty() { - std::fs::remove_file(&path)?; - } else { - atomic_write(&path, without.as_bytes())?; - } - } - Ok(Some(InstallChange { - path: relative.to_string(), - action: ChangeAction::Removed, - })) -} - -fn write_managed( - root: &Path, - relative: &str, - bytes: &[u8], - dry_run: bool, -) -> Result { - ensure_safe_managed_path(root, relative)?; - let path = root.join(relative); - let existed = path.exists(); - let action = classify_bytes(&path, bytes)?; - if !dry_run && action != ChangeAction::Unchanged { - atomic_write(&path, bytes)?; - } - Ok(InstallChange { - path: relative.to_string(), - action: if existed { - action - } else { - ChangeAction::Created - }, - }) -} - -fn classify_bytes(path: &Path, wanted: &[u8]) -> Result { - match std::fs::read(path) { - Ok(existing) if existing == wanted => Ok(ChangeAction::Unchanged), - Ok(_) => Ok(ChangeAction::Updated), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(ChangeAction::Created), - Err(error) => Err(error.into()), - } -} - -fn ensure_safe_managed_path(root: &Path, relative: &str) -> Result<(), TexoError> { - let relative_path = Path::new(relative); - if relative_path.is_absolute() - || relative_path.components().any(|component| { - !matches!( - component, - std::path::Component::Normal(_) | std::path::Component::CurDir - ) - }) - { - return Err(config_error( - relative, - "managed path must remain below the workspace root", - )); - } - let mut current = root.to_path_buf(); - for component in relative_path.components() { - if let std::path::Component::Normal(name) = component { - current.push(name); - match std::fs::symlink_metadata(¤t) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(config_error( - relative, - &format!("managed path crosses symbolic link `{}`", current.display()), - )); - } - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - } - } - Ok(()) -} - -fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), TexoError> { - let parent = path - .parent() - .ok_or_else(|| config_error(&path.display().to_string(), "managed path has no parent"))?; - std::fs::create_dir_all(parent)?; - let existing_permissions = std::fs::symlink_metadata(path) - .ok() - .filter(|metadata| metadata.file_type().is_file()) - .map(|metadata| metadata.permissions()); - let name = path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| config_error(&path.display().to_string(), "file name is not UTF-8"))?; - for _attempt in 0..100 { - let counter = INSTALL_TMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let tmp = parent.join(format!( - ".{name}.texo-install-{}-{counter}.tmp", - std::process::id() - )); - let mut file = match std::fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&tmp) - { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error.into()), - }; - let result = (|| -> std::io::Result<()> { - file.write_all(bytes)?; - if let Some(permissions) = &existing_permissions { - file.set_permissions(permissions.clone())?; - } - file.sync_all()?; - drop(file); - std::fs::rename(&tmp, path)?; - #[cfg(unix)] - std::fs::File::open(parent)?.sync_all()?; - Ok(()) - })(); - if result.is_err() { - let _removed = std::fs::remove_file(&tmp); - } - return result.map_err(Into::into); - } - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "could not allocate a private install staging file", - ) - .into()) -} - -fn strip_marked_block(input: &str, start: &str, end: &str) -> Result<(String, bool), TexoError> { - let Some(start_offset) = input.find(start) else { - if input.contains(end) { - return Err(config_error( - AGENT_GUIDE_PATH, - "orphaned managed end marker", - )); - } - return Ok((input.to_string(), false)); - }; - let Some(relative_end) = input[start_offset..].find(end) else { - return Err(config_error( - AGENT_GUIDE_PATH, - "unterminated managed marker block", - )); - }; - let end_offset = start_offset + relative_end + end.len(); - if input[end_offset..].contains(start) { - return Err(config_error( - AGENT_GUIDE_PATH, - "multiple managed marker blocks", - )); - } - let mut without = String::new(); - without.push_str(input[..start_offset].trim_end()); - if !without.is_empty() { - without.push('\n'); - } - without.push_str(input[end_offset..].trim_start_matches(['\r', '\n'])); - Ok((without, true)) -} - -fn append_block(existing: &str, block: &str) -> String { - let mut out = existing.trim_end().to_string(); - if !out.is_empty() { - out.push_str("\n\n"); - } - out.push_str(block); - out.push('\n'); - out -} - -fn read_optional_string(path: &Path) -> Result { - match std::fs::read_to_string(path) { - Ok(value) => Ok(value), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), - Err(error) => Err(error.into()), - } -} - -fn with_newline(mut bytes: Vec) -> Vec { - bytes.push(b'\n'); - bytes -} - -fn escape_toml(value: &str) -> String { - value.replace('\\', "\\\\").replace('"', "\\\"") -} - fn config_error(path: &str, detail: &str) -> TexoError { TexoError::Config { detail: format!("{path}: {detail}"), @@ -1138,242 +489,8 @@ fn config_error(path: &str, detail: &str) -> TexoError { /// Return the managed adapter paths for doctor diagnostics. #[must_use] pub fn adapter_paths() -> [PathBuf; 4] { - [ - PathBuf::from(MCP_MANIFEST_PATH), - PathBuf::from(CLAUDE_MCP_PATH), - PathBuf::from(CURSOR_MCP_PATH), - PathBuf::from(CODEX_CONFIG_PATH), - ] + adapter::paths() } #[cfg(test)] -mod tests { - use tempfile::TempDir; - - use super::*; - - #[test] - fn install_all_is_idempotent_and_uninstall_preserves_user_content() { - let dir = TempDir::new().expect("tempdir"); - std::fs::write(dir.path().join(AGENT_GUIDE_PATH), "# User guide\n").expect("user guide"); - std::fs::create_dir_all(dir.path().join(".codex")).expect("codex dir"); - std::fs::write(dir.path().join(CODEX_CONFIG_PATH), "model = \"example\"\n") - .expect("codex config"); - std::fs::write( - dir.path().join(CLAUDE_MCP_PATH), - b"{\"user\":true,\"mcpServers\":{}}", - ) - .expect("claude config"); - - install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); - let first = fingerprint_files(dir.path()); - let second = install(dir.path(), "demo", &[ClientTarget::All], false).expect("reinstall"); - assert!(second - .changes - .iter() - .all(|change| change.action == ChangeAction::Unchanged)); - assert_eq!(fingerprint_files(dir.path()), first); - - uninstall(dir.path(), &[], false).expect("uninstall"); - assert!(std::fs::read_to_string(dir.path().join(AGENT_GUIDE_PATH)) - .expect("guide") - .contains("# User guide")); - assert!(std::fs::read_to_string(dir.path().join(CODEX_CONFIG_PATH)) - .expect("codex") - .contains("model = \"example\"")); - let claude: Value = serde_json::from_slice( - &std::fs::read(dir.path().join(CLAUDE_MCP_PATH)).expect("claude"), - ) - .expect("valid json"); - assert_eq!(claude["user"], true); - assert!(claude["mcpServers"].get("texo").is_none()); - assert!(dir.path().join(".texo/config.toml").is_file()); - } - - #[test] - fn dry_run_is_write_free_on_a_fresh_root() { - let dir = TempDir::new().expect("tempdir"); - let report = install(dir.path(), "demo", &[ClientTarget::All], true).expect("preview"); - assert!(report.dry_run); - assert!(report - .changes - .iter() - .all(|change| change.action == ChangeAction::Created)); - assert!(fingerprint_files(dir.path()).is_empty()); - } - - #[test] - fn conflicting_adapter_fails_before_any_write() { - let dir = TempDir::new().expect("tempdir"); - let conflict = br#"{"mcpServers":{"texo":{"command":"other"}}}"#; - std::fs::write(dir.path().join(CLAUDE_MCP_PATH), conflict).expect("conflict"); - let before = fingerprint_files(dir.path()); - - let error = install(dir.path(), "demo", &[ClientTarget::All], false) - .expect_err("conflict must fail"); - - assert!(error.to_string().contains("not managed")); - assert_eq!(fingerprint_files(dir.path()), before); - } - - #[test] - fn uninstall_refuses_an_unmanaged_texo_entry() { - let dir = TempDir::new().expect("tempdir"); - let conflict = br#"{"mcpServers":{"texo":{"command":"other"}}}"#; - std::fs::write(dir.path().join(CLAUDE_MCP_PATH), conflict).expect("conflict"); - - let error = uninstall(dir.path(), &[], false).expect_err("unmanaged entry must survive"); - - assert!(error.to_string().contains("not managed")); - assert_eq!( - std::fs::read(dir.path().join(CLAUDE_MCP_PATH)).expect("preserved"), - conflict - ); - } - - #[test] - fn uninstall_deletes_only_empty_files_created_by_texo() { - let created = TempDir::new().expect("created root"); - install(created.path(), "demo", &[ClientTarget::All], false).expect("install created"); - uninstall(created.path(), &[], false).expect("uninstall created"); - for relative in [ - CLAUDE_MCP_PATH, - CURSOR_MCP_PATH, - CODEX_CONFIG_PATH, - AGENT_GUIDE_PATH, - ] { - assert!( - !created.path().join(relative).exists(), - "{relative} removed" - ); - } - assert!(created.path().join(".texo/config.toml").is_file()); - - let existing = TempDir::new().expect("existing root"); - std::fs::create_dir_all(existing.path().join(".cursor")).expect("cursor dir"); - std::fs::create_dir_all(existing.path().join(".codex")).expect("codex dir"); - std::fs::write(existing.path().join(CLAUDE_MCP_PATH), "{}\n").expect("claude"); - std::fs::write(existing.path().join(CURSOR_MCP_PATH), "{}\n").expect("cursor"); - std::fs::write(existing.path().join(CODEX_CONFIG_PATH), "").expect("codex"); - std::fs::write(existing.path().join(AGENT_GUIDE_PATH), "").expect("guide"); - install(existing.path(), "demo", &[ClientTarget::All], false).expect("install existing"); - uninstall(existing.path(), &[], false).expect("uninstall existing"); - for relative in [ - CLAUDE_MCP_PATH, - CURSOR_MCP_PATH, - CODEX_CONFIG_PATH, - AGENT_GUIDE_PATH, - ] { - assert!( - existing.path().join(relative).is_file(), - "{relative} preserved" - ); - } - } - - #[test] - fn targeted_uninstall_keeps_shared_and_other_client_entries() { - let dir = TempDir::new().expect("tempdir"); - install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); - - let report = uninstall(dir.path(), &[ClientTarget::Claude], false).expect("uninstall"); - - assert_eq!(report.clients, vec![ClientTarget::Claude]); - assert!(!dir.path().join(CLAUDE_MCP_PATH).exists()); - assert!(dir.path().join(CURSOR_MCP_PATH).is_file()); - assert!(dir.path().join(CODEX_CONFIG_PATH).is_file()); - assert!(dir.path().join(MCP_MANIFEST_PATH).is_file()); - assert!(dir.path().join(crate::hooks::HOOKS_MANIFEST_PATH).is_file()); - assert!(dir.path().join(AGENT_GUIDE_PATH).is_file()); - } - - #[test] - fn json_merge_preserves_user_key_order() { - let dir = TempDir::new().expect("tempdir"); - std::fs::write( - dir.path().join(CLAUDE_MCP_PATH), - r#"{"zeta":1,"alpha":2,"mcpServers":{"other":{"command":"other"}}}"#, - ) - .expect("config"); - - install(dir.path(), "demo", &[ClientTarget::Claude], false).expect("install"); - - let merged = std::fs::read_to_string(dir.path().join(CLAUDE_MCP_PATH)).expect("merged"); - let zeta = merged.find("\"zeta\"").expect("zeta"); - let alpha = merged.find("\"alpha\"").expect("alpha"); - let servers = merged.find("\"mcpServers\"").expect("servers"); - assert!(zeta < alpha && alpha < servers); - } - - #[test] - fn uninstall_conflict_is_detected_before_any_removal() { - let dir = TempDir::new().expect("tempdir"); - install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); - std::fs::write( - dir.path().join(CURSOR_MCP_PATH), - r#"{"mcpServers":{"texo":{"command":"other"}}}"#, - ) - .expect("conflict"); - let before = fingerprint_files(dir.path()); - - let error = uninstall(dir.path(), &[], false).expect_err("conflict"); - - assert!(error.to_string().contains("not managed")); - assert_eq!(fingerprint_files(dir.path()), before); - } - - #[cfg(unix)] - #[test] - fn install_refuses_symlinked_client_paths_and_preserves_permissions() { - use std::os::unix::fs::{symlink, PermissionsExt as _}; - - let linked = TempDir::new().expect("linked root"); - let outside = TempDir::new().expect("outside"); - symlink(outside.path(), linked.path().join(".cursor")).expect("symlink"); - let before = fingerprint_files(outside.path()); - let error = install(linked.path(), "demo", &[ClientTarget::All], false) - .expect_err("symlink must fail"); - assert!(error.to_string().contains("symbolic link")); - assert_eq!(fingerprint_files(outside.path()), before); - assert!(!linked.path().join(".texo").exists()); - - let permissions = TempDir::new().expect("permissions root"); - std::fs::write(permissions.path().join(CLAUDE_MCP_PATH), "{}\n").expect("config"); - let mut mode = std::fs::metadata(permissions.path().join(CLAUDE_MCP_PATH)) - .expect("metadata") - .permissions(); - mode.set_mode(0o600); - std::fs::set_permissions(permissions.path().join(CLAUDE_MCP_PATH), mode) - .expect("permissions"); - install(permissions.path(), "demo", &[ClientTarget::Claude], false).expect("install"); - assert_eq!( - std::fs::metadata(permissions.path().join(CLAUDE_MCP_PATH)) - .expect("metadata") - .permissions() - .mode() - & 0o777, - 0o600 - ); - } - - fn fingerprint_files(root: &Path) -> Vec<(String, Vec)> { - let mut rows = walkdir::WalkDir::new(root) - .into_iter() - .filter_map(Result::ok) - .filter(|entry| entry.file_type().is_file()) - .map(|entry| { - ( - entry - .path() - .strip_prefix(root) - .expect("relative") - .to_string_lossy() - .to_string(), - std::fs::read(entry.path()).expect("read"), - ) - }) - .collect::>(); - rows.sort_by(|left, right| left.0.cmp(&right.0)); - rows - } -} +mod tests; diff --git a/src/install/adapter.rs b/src/install/adapter.rs new file mode 100644 index 0000000..6a912a2 --- /dev/null +++ b/src/install/adapter.rs @@ -0,0 +1,473 @@ +//! Client adapter merge, removal, and managed-manifest policy. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use serde::Serialize; +use serde_json::{json, Value}; + +use crate::error::TexoError; + +use super::filesystem::{ + append_block, atomic_write, classify_bytes, ensure_safe_managed_path, escape_toml, + read_optional_string, remove_marked_block, strip_marked_block, with_newline, +}; +use super::{ + config_error, ChangeAction, ClientTarget, InstallChange, AGENT_GUIDE_PATH, MCP_MANIFEST_PATH, +}; + +pub(super) const CLAUDE_MCP_PATH: &str = ".mcp.json"; +pub(super) const CURSOR_MCP_PATH: &str = ".cursor/mcp.json"; +pub(super) const CODEX_CONFIG_PATH: &str = ".codex/config.toml"; +const CODEX_MARKER_START: &str = "# texo:install:codex:start"; +const CODEX_MARKER_END: &str = "# texo:install:codex:end"; +pub(super) const AGENT_MARKER_START: &str = ""; +pub(super) const AGENT_MARKER_END: &str = ""; + +type OrderedJsonObject = IndexMap>; + +pub(super) fn merge_client_adapter( + root: &Path, + workspace_id: &str, + client: ClientTarget, + journal_id: Option<&str>, + previous_managed: Option<&Value>, + dry_run: bool, +) -> Result, TexoError> { + let change = match client { + ClientTarget::Claude => merge_json_adapter( + root, + CLAUDE_MCP_PATH, + workspace_id, + journal_id, + previous_managed, + dry_run, + )?, + ClientTarget::Cursor => merge_json_adapter( + root, + CURSOR_MCP_PATH, + workspace_id, + journal_id, + previous_managed, + dry_run, + )?, + ClientTarget::Codex => merge_codex_adapter(root, workspace_id, journal_id, dry_run)?, + ClientTarget::Auto | ClientTarget::All => return Ok(None), + }; + Ok(Some(change)) +} + +pub(super) fn managed_server_entry(root: &Path) -> Result, TexoError> { + let path = root.join(MCP_MANIFEST_PATH); + if !path.exists() { + return Ok(None); + } + let manifest = serde_json::from_slice::(&std::fs::read(path)?)?; + if manifest.get("schema").and_then(Value::as_str) != Some("texo.mcp-install.v1") { + return Err(config_error( + MCP_MANIFEST_PATH, + "existing manifest is not owned by this installer", + )); + } + Ok(manifest.get("server").cloned()) +} + +pub(super) fn resolve_clients(root: &Path, requested: &[ClientTarget]) -> Vec { + let requested = if requested.is_empty() { + &[ClientTarget::Auto][..] + } else { + requested + }; + let mut selected = BTreeSet::new(); + for target in requested { + match target { + ClientTarget::All => { + selected.extend([ + ClientTarget::Codex, + ClientTarget::Claude, + ClientTarget::Cursor, + ]); + } + ClientTarget::Auto => { + if root.join(".codex").exists() { + selected.insert(ClientTarget::Codex); + } + if root.join(CLAUDE_MCP_PATH).exists() || root.join(".claude").exists() { + selected.insert(ClientTarget::Claude); + } + if root.join(".cursor").exists() { + selected.insert(ClientTarget::Cursor); + } + } + concrete @ (ClientTarget::Codex | ClientTarget::Claude | ClientTarget::Cursor) => { + selected.insert(*concrete); + } + } + } + selected.into_iter().collect() +} + +pub(super) fn canonical_manifest( + workspace_id: &str, + journal_id: Option<&str>, + created_paths: &BTreeSet, +) -> Value { + json!({ + "schema": "texo.mcp-install.v1", + "server": server_entry(workspace_id, journal_id), + "created_paths": created_paths + }) +} + +fn server_entry(workspace_id: &str, journal_id: Option<&str>) -> Value { + let mut args = vec![ + "--root".to_string(), + ".".to_string(), + "--workspace".to_string(), + workspace_id.to_string(), + ]; + if let Some(journal_id) = journal_id { + args.push("--journal".to_string()); + args.push(journal_id.to_string()); + } + args.push("mcp".to_string()); + json!({ + "command": "texo", + "args": args, + "env": {} + }) +} + +fn merge_json_adapter( + root: &Path, + relative: &str, + workspace_id: &str, + journal_id: Option<&str>, + previous_managed: Option<&Value>, + dry_run: bool, +) -> Result { + ensure_safe_managed_path(root, relative)?; + let path = root.join(relative); + let existed = path.exists(); + let (mut document, mut servers) = read_ordered_adapter(&path, existed, relative)?; + let wanted = server_entry(workspace_id, journal_id); + if let Some(existing) = servers.get("texo") { + let existing = serde_json::from_str::(existing.get())?; + if existing != wanted && previous_managed != Some(&existing) { + return Err(config_error( + relative, + "existing mcpServers.texo is not managed by this installer", + )); + } + } + servers.insert("texo".to_string(), raw_json(&wanted)?); + document.insert("mcpServers".to_string(), raw_json(&servers)?); + let bytes = with_newline(serde_json::to_vec_pretty(&document)?); + let action = classify_bytes(&path, &bytes)?; + if !dry_run && action != ChangeAction::Unchanged { + atomic_write(&path, &bytes)?; + } + Ok(InstallChange { + path: relative.to_string(), + action: if !existed && action == ChangeAction::Updated { + ChangeAction::Created + } else { + action + }, + }) +} + +fn merge_codex_adapter( + root: &Path, + workspace_id: &str, + journal_id: Option<&str>, + dry_run: bool, +) -> Result { + ensure_safe_managed_path(root, CODEX_CONFIG_PATH)?; + let path = root.join(CODEX_CONFIG_PATH); + let existing = read_optional_string(&path)?; + let (without, had_marker) = + strip_marked_block(&existing, CODEX_MARKER_START, CODEX_MARKER_END)?; + if !without.trim().is_empty() { + let parsed = without + .parse::() + .map_err(|error| TexoError::Config { + detail: format!("{CODEX_CONFIG_PATH}: {error}"), + source: Some(Box::new(error)), + })?; + if parsed + .get("mcp_servers") + .and_then(|value| value.get("texo")) + .is_some() + { + return Err(config_error( + CODEX_CONFIG_PATH, + "existing mcp_servers.texo is outside the managed block", + )); + } + } + let journal_args = journal_id.map_or_else(String::new, |journal_id| { + format!(", \"--journal\", \"{}\"", escape_toml(journal_id)) + }); + let args = format!( + "[\"--root\", \".\", \"--workspace\", \"{}\"{journal_args}, \"mcp\"]", + escape_toml(workspace_id) + ); + let block = format!( + "{CODEX_MARKER_START}\n[mcp_servers.texo]\ncommand = \"texo\"\nargs = {args}\n{CODEX_MARKER_END}" + ); + let updated = append_block(&without, &block); + let bytes = updated.into_bytes(); + let action = classify_bytes(&path, &bytes)?; + if !dry_run && action != ChangeAction::Unchanged { + atomic_write(&path, &bytes)?; + } + Ok(InstallChange { + path: CODEX_CONFIG_PATH.to_string(), + action: if !path.exists() && !had_marker { + ChangeAction::Created + } else { + action + }, + }) +} + +pub(super) fn upsert_agent_guide( + root: &Path, + workspace_id: &str, + dry_run: bool, +) -> Result { + ensure_safe_managed_path(root, AGENT_GUIDE_PATH)?; + let path = root.join(AGENT_GUIDE_PATH); + let existing = read_optional_string(&path)?; + let (without, _) = strip_marked_block(&existing, AGENT_MARKER_START, AGENT_MARKER_END)?; + let block = format!( + "{AGENT_MARKER_START}\n## Texo agent context\n\nWorkspace: `{workspace_id}`. Start with the `get_agent_context` MCP tool before answering from project knowledge. Reuse its snapshot token with `search_knowledge`, `explain_knowledge`, and `triangulate` so one investigation stays on one frontier. Inspect coverage before treating absence as evidence. Absence of a relation verdict never means unrelated. Texo MCP tools are local and read-only.\n{AGENT_MARKER_END}" + ); + let bytes = append_block(&without, &block).into_bytes(); + let action = classify_bytes(&path, &bytes)?; + if !dry_run && action != ChangeAction::Unchanged { + atomic_write(&path, &bytes)?; + } + Ok(InstallChange { + path: AGENT_GUIDE_PATH.to_string(), + action: if path.exists() { + action + } else { + ChangeAction::Created + }, + }) +} + +pub(super) fn client_path(client: ClientTarget) -> Option<&'static str> { + match client { + ClientTarget::Codex => Some(CODEX_CONFIG_PATH), + ClientTarget::Claude => Some(CLAUDE_MCP_PATH), + ClientTarget::Cursor => Some(CURSOR_MCP_PATH), + ClientTarget::Auto | ClientTarget::All => None, + } +} + +fn managed_manifest(root: &Path) -> Result, TexoError> { + ensure_safe_managed_path(root, MCP_MANIFEST_PATH)?; + let path = root.join(MCP_MANIFEST_PATH); + if !path.exists() { + return Ok(None); + } + let document = serde_json::from_slice::(&std::fs::read(path)?)?; + if document.get("schema").and_then(Value::as_str) != Some("texo.mcp-install.v1") { + return Err(config_error( + MCP_MANIFEST_PATH, + "file is not managed by this installer", + )); + } + Ok(Some(document)) +} + +pub(super) fn managed_created_paths(root: &Path) -> Result, TexoError> { + let Some(document) = managed_manifest(root)? else { + return Ok(BTreeSet::new()); + }; + document.get("created_paths").map_or_else( + || Ok(BTreeSet::new()), + |paths| serde_json::from_value(paths.clone()).map_err(TexoError::Json), + ) +} + +pub(super) fn managed_workspace_id(root: &Path) -> Result, TexoError> { + let Some(document) = managed_manifest(root)? else { + return Ok(None); + }; + // Recover the workspace from the `--workspace ` flag pair, not a fixed + // positional index. A managed manifest with a valid schema but an + // unexpected args shape must fail closed rather than silently rewrite the + // manifest to point future MCP clients at the `demo` workspace. + let args = document + .get("server") + .and_then(|server| server.get("args")) + .and_then(Value::as_array) + .ok_or_else(|| config_error(MCP_MANIFEST_PATH, "managed manifest has no server args"))?; + let workspace = args.windows(2).find_map(|pair| { + if pair[0].as_str() == Some("--workspace") { + pair[1].as_str() + } else { + None + } + }); + match workspace { + Some(id) => Ok(Some(id.to_string())), + None => Err(config_error( + MCP_MANIFEST_PATH, + "managed manifest args carry no recoverable --workspace id; refusing to rewrite", + )), + } +} + +pub(super) fn managed_journal_id(root: &Path) -> Result, TexoError> { + let Some(document) = managed_manifest(root)? else { + return Ok(None); + }; + let args = document + .get("server") + .and_then(|server| server.get("args")) + .and_then(Value::as_array) + .ok_or_else(|| config_error(MCP_MANIFEST_PATH, "managed manifest has no server args"))?; + Ok(args.windows(2).find_map(|pair| { + if pair[0].as_str() == Some("--journal") { + pair[1].as_str().map(str::to_string) + } else { + None + } + })) +} + +pub(super) fn preflight_remove_client( + root: &Path, + client: ClientTarget, + created_paths: &BTreeSet, +) -> Result<(), TexoError> { + let _change = remove_client(root, client, created_paths, true)?; + Ok(()) +} + +pub(super) fn remove_client( + root: &Path, + client: ClientTarget, + created_paths: &BTreeSet, + dry_run: bool, +) -> Result, TexoError> { + let Some(relative) = client_path(client) else { + return Ok(None); + }; + let remove_empty = created_paths.contains(relative); + match client { + ClientTarget::Claude | ClientTarget::Cursor => { + remove_json_adapter(root, relative, remove_empty, dry_run) + } + ClientTarget::Codex => remove_marked_block( + root, + relative, + CODEX_MARKER_START, + CODEX_MARKER_END, + remove_empty, + dry_run, + ), + ClientTarget::Auto | ClientTarget::All => Ok(None), + } +} + +fn remove_json_adapter( + root: &Path, + relative: &str, + remove_empty: bool, + dry_run: bool, +) -> Result, TexoError> { + ensure_safe_managed_path(root, relative)?; + let path = root.join(relative); + if !path.exists() { + return Ok(None); + } + let (mut document, mut servers) = read_ordered_adapter(&path, true, relative)?; + let Some(existing) = servers.get("texo") else { + return Ok(None); + }; + if !is_managed_server_entry(&serde_json::from_str::(existing.get())?) { + return Err(config_error( + relative, + "mcpServers.texo is not managed by this installer", + )); + } + servers.shift_remove("texo"); + document.insert("mcpServers".to_string(), raw_json(&servers)?); + if !dry_run { + if remove_empty && json_adapter_is_empty(&document, &servers) { + std::fs::remove_file(&path)?; + } else { + atomic_write(&path, &with_newline(serde_json::to_vec_pretty(&document)?))?; + } + } + Ok(Some(InstallChange { + path: relative.to_string(), + action: ChangeAction::Removed, + })) +} + +fn read_ordered_adapter( + path: &Path, + existed: bool, + relative: &str, +) -> Result<(OrderedJsonObject, OrderedJsonObject), TexoError> { + let document = if existed { + serde_json::from_slice::(&std::fs::read(path)?)? + } else { + OrderedJsonObject::new() + }; + let servers = document.get("mcpServers").map_or_else( + || Ok(OrderedJsonObject::new()), + |raw| { + serde_json::from_str::(raw.get()).map_err(|error| { + TexoError::Config { + detail: format!("{relative}: mcpServers must be an object"), + source: Some(Box::new(error)), + } + }) + }, + )?; + Ok((document, servers)) +} + +fn raw_json(value: &T) -> Result, TexoError> { + serde_json::value::RawValue::from_string(serde_json::to_string(value)?).map_err(TexoError::Json) +} + +fn json_adapter_is_empty(document: &OrderedJsonObject, servers: &OrderedJsonObject) -> bool { + document.len() == 1 && servers.is_empty() +} + +fn is_managed_server_entry(value: &Value) -> bool { + value.get("command").and_then(Value::as_str) == Some("texo") + && value + .get("args") + .and_then(Value::as_array) + .is_some_and(|args| { + args.first().and_then(Value::as_str) == Some("--root") + && args.get(1).and_then(Value::as_str) == Some(".") + && args.get(2).and_then(Value::as_str) == Some("--workspace") + && args.get(3).and_then(Value::as_str).is_some() + && args.last().and_then(Value::as_str) == Some("mcp") + && (args.len() == 5 + || (args.len() == 7 + && args.get(4).and_then(Value::as_str) == Some("--journal") + && args.get(5).and_then(Value::as_str).is_some())) + }) +} + +pub(super) fn paths() -> [PathBuf; 4] { + [ + PathBuf::from(MCP_MANIFEST_PATH), + PathBuf::from(CLAUDE_MCP_PATH), + PathBuf::from(CURSOR_MCP_PATH), + PathBuf::from(CODEX_CONFIG_PATH), + ] +} diff --git a/src/install/entry.rs b/src/install/entry.rs new file mode 100644 index 0000000..48afde8 --- /dev/null +++ b/src/install/entry.rs @@ -0,0 +1,21 @@ +//! Public installation entry point. + +use std::path::Path; + +use crate::error::TexoError; + +use super::{ClientTarget, InstallReport}; + +/// Install the lightweight Texo appliance. +/// +/// # Errors +/// Returns an error when existing client configuration is malformed, already +/// owns a conflicting `texo` entry, or a managed write fails. +pub fn install( + root: &Path, + workspace_id: &str, + requested: &[ClientTarget], + dry_run: bool, +) -> Result { + super::install_for_journal(root, workspace_id, None, requested, dry_run) +} diff --git a/src/install/filesystem.rs b/src/install/filesystem.rs new file mode 100644 index 0000000..c3e50ba --- /dev/null +++ b/src/install/filesystem.rs @@ -0,0 +1,252 @@ +//! Managed-path validation and crash-safe file mutation primitives. + +use std::io::Write as _; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::Value; + +use crate::error::TexoError; + +use super::{config_error, ChangeAction, InstallChange, AGENT_GUIDE_PATH}; + +static INSTALL_TMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +pub(super) fn remove_managed_file( + root: &Path, + relative: &str, + schema: &str, + dry_run: bool, +) -> Result { + ensure_safe_managed_path(root, relative)?; + let path = root.join(relative); + let action = if path.is_file() { + let document = serde_json::from_slice::(&std::fs::read(&path)?)?; + if document.get("schema").and_then(Value::as_str) != Some(schema) { + return Err(config_error( + relative, + "file is not managed by this installer", + )); + } + if !dry_run { + std::fs::remove_file(path)?; + } + ChangeAction::Removed + } else { + ChangeAction::Unchanged + }; + Ok(InstallChange { + path: relative.to_string(), + action, + }) +} + +pub(super) fn remove_marked_block( + root: &Path, + relative: &str, + start: &str, + end: &str, + remove_empty: bool, + dry_run: bool, +) -> Result, TexoError> { + ensure_safe_managed_path(root, relative)?; + let path = root.join(relative); + let existing = read_optional_string(&path)?; + let (without, had_marker) = strip_marked_block(&existing, start, end)?; + if !had_marker { + return Ok(None); + } + if !dry_run { + if remove_empty && without.trim().is_empty() { + std::fs::remove_file(&path)?; + } else { + atomic_write(&path, without.as_bytes())?; + } + } + Ok(Some(InstallChange { + path: relative.to_string(), + action: ChangeAction::Removed, + })) +} + +pub(super) fn write_managed( + root: &Path, + relative: &str, + bytes: &[u8], + dry_run: bool, +) -> Result { + ensure_safe_managed_path(root, relative)?; + let path = root.join(relative); + let existed = path.exists(); + let action = classify_bytes(&path, bytes)?; + if !dry_run && action != ChangeAction::Unchanged { + atomic_write(&path, bytes)?; + } + Ok(InstallChange { + path: relative.to_string(), + action: if existed { + action + } else { + ChangeAction::Created + }, + }) +} + +pub(super) fn classify_bytes(path: &Path, wanted: &[u8]) -> Result { + match std::fs::read(path) { + Ok(existing) if existing == wanted => Ok(ChangeAction::Unchanged), + Ok(_) => Ok(ChangeAction::Updated), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(ChangeAction::Created), + Err(error) => Err(error.into()), + } +} + +pub(super) fn ensure_safe_managed_path(root: &Path, relative: &str) -> Result<(), TexoError> { + let relative_path = Path::new(relative); + if relative_path.is_absolute() + || relative_path.components().any(|component| { + !matches!( + component, + std::path::Component::Normal(_) | std::path::Component::CurDir + ) + }) + { + return Err(config_error( + relative, + "managed path must remain below the workspace root", + )); + } + let mut current = root.to_path_buf(); + for component in relative_path.components() { + if let std::path::Component::Normal(name) = component { + current.push(name); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(config_error( + relative, + &format!("managed path crosses symbolic link `{}`", current.display()), + )); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + } + Ok(()) +} + +pub(super) fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), TexoError> { + let parent = path + .parent() + .ok_or_else(|| config_error(&path.display().to_string(), "managed path has no parent"))?; + std::fs::create_dir_all(parent)?; + let existing_permissions = std::fs::symlink_metadata(path) + .ok() + .filter(|metadata| metadata.file_type().is_file()) + .map(|metadata| metadata.permissions()); + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| config_error(&path.display().to_string(), "file name is not UTF-8"))?; + for _attempt in 0..100 { + let counter = INSTALL_TMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!( + ".{name}.texo-install-{}-{counter}.tmp", + std::process::id() + )); + let mut file = match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&tmp) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + }; + let result = (|| -> std::io::Result<()> { + file.write_all(bytes)?; + if let Some(permissions) = &existing_permissions { + file.set_permissions(permissions.clone())?; + } + file.sync_all()?; + drop(file); + std::fs::rename(&tmp, path)?; + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _removed = std::fs::remove_file(&tmp); + } + return result.map_err(Into::into); + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not allocate a private install staging file", + ) + .into()) +} + +pub(super) fn strip_marked_block( + input: &str, + start: &str, + end: &str, +) -> Result<(String, bool), TexoError> { + let Some(start_offset) = input.find(start) else { + if input.contains(end) { + return Err(config_error( + AGENT_GUIDE_PATH, + "orphaned managed end marker", + )); + } + return Ok((input.to_string(), false)); + }; + let Some(relative_end) = input[start_offset..].find(end) else { + return Err(config_error( + AGENT_GUIDE_PATH, + "unterminated managed marker block", + )); + }; + let end_offset = start_offset + relative_end + end.len(); + if input[end_offset..].contains(start) { + return Err(config_error( + AGENT_GUIDE_PATH, + "multiple managed marker blocks", + )); + } + let mut without = String::new(); + without.push_str(input[..start_offset].trim_end()); + if !without.is_empty() { + without.push('\n'); + } + without.push_str(input[end_offset..].trim_start_matches(['\r', '\n'])); + Ok((without, true)) +} + +pub(super) fn append_block(existing: &str, block: &str) -> String { + let mut out = existing.trim_end().to_string(); + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(block); + out.push('\n'); + out +} + +pub(super) fn read_optional_string(path: &Path) -> Result { + match std::fs::read_to_string(path) { + Ok(value) => Ok(value), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(String::new()), + Err(error) => Err(error.into()), + } +} + +pub(super) fn with_newline(mut bytes: Vec) -> Vec { + bytes.push(b'\n'); + bytes +} + +pub(super) fn escape_toml(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} diff --git a/src/install/model.rs b/src/install/model.rs new file mode 100644 index 0000000..fb9ea09 --- /dev/null +++ b/src/install/model.rs @@ -0,0 +1,33 @@ +//! Stable installer reports re-exported by the install facade. + +use serde::Serialize; + +use super::{ChangeAction, ClientJournalRoute, ClientTarget}; + +/// One install/uninstall path result. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct InstallChange { + /// Workspace-relative path. + pub path: String, + /// Action applied or planned. + pub action: ChangeAction, +} + +/// Machine-readable appliance installation report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct InstallReport { + /// Report schema. + pub schema: &'static str, + /// Workspace root. + pub root: String, + /// Workspace id installed. + pub workspace_id: String, + /// Whether this was a write-free preview. + pub dry_run: bool, + /// Selected concrete clients. + pub clients: Vec, + /// Physical read journal selected for each client adapter. + pub routes: Vec, + /// Ordered path changes. + pub changes: Vec, +} diff --git a/src/install/tests.rs b/src/install/tests.rs new file mode 100644 index 0000000..3f5b773 --- /dev/null +++ b/src/install/tests.rs @@ -0,0 +1,230 @@ +use std::path::Path; + +use serde_json::Value; +use tempfile::TempDir; + +use super::adapter::{CLAUDE_MCP_PATH, CODEX_CONFIG_PATH, CURSOR_MCP_PATH}; +use super::{install, uninstall, ChangeAction, ClientTarget, AGENT_GUIDE_PATH, MCP_MANIFEST_PATH}; + +#[test] +fn install_all_is_idempotent_and_uninstall_preserves_user_content() { + let dir = TempDir::new().expect("tempdir"); + std::fs::write(dir.path().join(AGENT_GUIDE_PATH), "# User guide\n").expect("user guide"); + std::fs::create_dir_all(dir.path().join(".codex")).expect("codex dir"); + std::fs::write(dir.path().join(CODEX_CONFIG_PATH), "model = \"example\"\n") + .expect("codex config"); + std::fs::write( + dir.path().join(CLAUDE_MCP_PATH), + b"{\"user\":true,\"mcpServers\":{}}", + ) + .expect("claude config"); + + install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); + let first = fingerprint_files(dir.path()); + let second = install(dir.path(), "demo", &[ClientTarget::All], false).expect("reinstall"); + assert!(second + .changes + .iter() + .all(|change| change.action == ChangeAction::Unchanged)); + assert_eq!(fingerprint_files(dir.path()), first); + + uninstall(dir.path(), &[], false).expect("uninstall"); + assert!(std::fs::read_to_string(dir.path().join(AGENT_GUIDE_PATH)) + .expect("guide") + .contains("# User guide")); + assert!(std::fs::read_to_string(dir.path().join(CODEX_CONFIG_PATH)) + .expect("codex") + .contains("model = \"example\"")); + let claude: Value = + serde_json::from_slice(&std::fs::read(dir.path().join(CLAUDE_MCP_PATH)).expect("claude")) + .expect("valid json"); + assert_eq!(claude["user"], true); + assert!(claude["mcpServers"].get("texo").is_none()); + assert!(dir.path().join(".texo/config.toml").is_file()); +} + +#[test] +fn dry_run_is_write_free_on_a_fresh_root() { + let dir = TempDir::new().expect("tempdir"); + let report = install(dir.path(), "demo", &[ClientTarget::All], true).expect("preview"); + assert!(report.dry_run); + assert!(report + .changes + .iter() + .all(|change| change.action == ChangeAction::Created)); + assert!(fingerprint_files(dir.path()).is_empty()); +} + +#[test] +fn conflicting_adapter_fails_before_any_write() { + let dir = TempDir::new().expect("tempdir"); + let conflict = br#"{"mcpServers":{"texo":{"command":"other"}}}"#; + std::fs::write(dir.path().join(CLAUDE_MCP_PATH), conflict).expect("conflict"); + let before = fingerprint_files(dir.path()); + + let error = + install(dir.path(), "demo", &[ClientTarget::All], false).expect_err("conflict must fail"); + + assert!(error.to_string().contains("not managed")); + assert_eq!(fingerprint_files(dir.path()), before); +} + +#[test] +fn uninstall_refuses_an_unmanaged_texo_entry() { + let dir = TempDir::new().expect("tempdir"); + let conflict = br#"{"mcpServers":{"texo":{"command":"other"}}}"#; + std::fs::write(dir.path().join(CLAUDE_MCP_PATH), conflict).expect("conflict"); + + let error = uninstall(dir.path(), &[], false).expect_err("unmanaged entry must survive"); + + assert!(error.to_string().contains("not managed")); + assert_eq!( + std::fs::read(dir.path().join(CLAUDE_MCP_PATH)).expect("preserved"), + conflict + ); +} + +#[test] +fn uninstall_deletes_only_empty_files_created_by_texo() { + let created = TempDir::new().expect("created root"); + install(created.path(), "demo", &[ClientTarget::All], false).expect("install created"); + uninstall(created.path(), &[], false).expect("uninstall created"); + for relative in [ + CLAUDE_MCP_PATH, + CURSOR_MCP_PATH, + CODEX_CONFIG_PATH, + AGENT_GUIDE_PATH, + ] { + assert!( + !created.path().join(relative).exists(), + "{relative} removed" + ); + } + assert!(created.path().join(".texo/config.toml").is_file()); + + let existing = TempDir::new().expect("existing root"); + std::fs::create_dir_all(existing.path().join(".cursor")).expect("cursor dir"); + std::fs::create_dir_all(existing.path().join(".codex")).expect("codex dir"); + std::fs::write(existing.path().join(CLAUDE_MCP_PATH), "{}\n").expect("claude"); + std::fs::write(existing.path().join(CURSOR_MCP_PATH), "{}\n").expect("cursor"); + std::fs::write(existing.path().join(CODEX_CONFIG_PATH), "").expect("codex"); + std::fs::write(existing.path().join(AGENT_GUIDE_PATH), "").expect("guide"); + install(existing.path(), "demo", &[ClientTarget::All], false).expect("install existing"); + uninstall(existing.path(), &[], false).expect("uninstall existing"); + for relative in [ + CLAUDE_MCP_PATH, + CURSOR_MCP_PATH, + CODEX_CONFIG_PATH, + AGENT_GUIDE_PATH, + ] { + assert!( + existing.path().join(relative).is_file(), + "{relative} preserved" + ); + } +} + +#[test] +fn targeted_uninstall_keeps_shared_and_other_client_entries() { + let dir = TempDir::new().expect("tempdir"); + install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); + + let report = uninstall(dir.path(), &[ClientTarget::Claude], false).expect("uninstall"); + + assert_eq!(report.clients, vec![ClientTarget::Claude]); + assert!(!dir.path().join(CLAUDE_MCP_PATH).exists()); + assert!(dir.path().join(CURSOR_MCP_PATH).is_file()); + assert!(dir.path().join(CODEX_CONFIG_PATH).is_file()); + assert!(dir.path().join(MCP_MANIFEST_PATH).is_file()); + assert!(dir.path().join(crate::hooks::HOOKS_MANIFEST_PATH).is_file()); + assert!(dir.path().join(AGENT_GUIDE_PATH).is_file()); +} + +#[test] +fn json_merge_preserves_user_key_order() { + let dir = TempDir::new().expect("tempdir"); + std::fs::write( + dir.path().join(CLAUDE_MCP_PATH), + r#"{"zeta":1,"alpha":2,"mcpServers":{"other":{"command":"other"}}}"#, + ) + .expect("config"); + + install(dir.path(), "demo", &[ClientTarget::Claude], false).expect("install"); + + let merged = std::fs::read_to_string(dir.path().join(CLAUDE_MCP_PATH)).expect("merged"); + let zeta = merged.find("\"zeta\"").expect("zeta"); + let alpha = merged.find("\"alpha\"").expect("alpha"); + let servers = merged.find("\"mcpServers\"").expect("servers"); + assert!(zeta < alpha && alpha < servers); +} + +#[test] +fn uninstall_conflict_is_detected_before_any_removal() { + let dir = TempDir::new().expect("tempdir"); + install(dir.path(), "demo", &[ClientTarget::All], false).expect("install"); + std::fs::write( + dir.path().join(CURSOR_MCP_PATH), + r#"{"mcpServers":{"texo":{"command":"other"}}}"#, + ) + .expect("conflict"); + let before = fingerprint_files(dir.path()); + + let error = uninstall(dir.path(), &[], false).expect_err("conflict"); + + assert!(error.to_string().contains("not managed")); + assert_eq!(fingerprint_files(dir.path()), before); +} + +#[cfg(unix)] +#[test] +fn install_refuses_symlinked_client_paths_and_preserves_permissions() { + use std::os::unix::fs::{symlink, PermissionsExt as _}; + + let linked = TempDir::new().expect("linked root"); + let outside = TempDir::new().expect("outside"); + symlink(outside.path(), linked.path().join(".cursor")).expect("symlink"); + let before = fingerprint_files(outside.path()); + let error = + install(linked.path(), "demo", &[ClientTarget::All], false).expect_err("symlink must fail"); + assert!(error.to_string().contains("symbolic link")); + assert_eq!(fingerprint_files(outside.path()), before); + assert!(!linked.path().join(".texo").exists()); + + let permissions = TempDir::new().expect("permissions root"); + std::fs::write(permissions.path().join(CLAUDE_MCP_PATH), "{}\n").expect("config"); + let mut mode = std::fs::metadata(permissions.path().join(CLAUDE_MCP_PATH)) + .expect("metadata") + .permissions(); + mode.set_mode(0o600); + std::fs::set_permissions(permissions.path().join(CLAUDE_MCP_PATH), mode).expect("permissions"); + install(permissions.path(), "demo", &[ClientTarget::Claude], false).expect("install"); + assert_eq!( + std::fs::metadata(permissions.path().join(CLAUDE_MCP_PATH)) + .expect("metadata") + .permissions() + .mode() + & 0o777, + 0o600 + ); +} + +fn fingerprint_files(root: &Path) -> Vec<(String, Vec)> { + let mut rows = walkdir::WalkDir::new(root) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| { + ( + entry + .path() + .strip_prefix(root) + .expect("relative") + .to_string_lossy() + .to_string(), + std::fs::read(entry.path()).expect("read"), + ) + }) + .collect::>(); + rows.sort_by(|left, right| left.0.cmp(&right.0)); + rows +} diff --git a/src/knowledge.rs b/src/knowledge.rs index 0ef7e6f..f8d57d3 100644 --- a/src/knowledge.rs +++ b/src/knowledge.rs @@ -4,57 +4,16 @@ use std::fmt; use serde::{Deserialize, Serialize}; +use crate::events::ids::{blake3_hash_hex, WorkspaceId}; use crate::topology::JournalId; -use thiserror::Error; -use crate::events::ids::{blake3_hash_hex, WorkspaceId}; +mod contracts; + +pub use contracts::{KnowledgeContractError, KnowledgeCoverage}; /// Maximum exact evidence excerpt carried by one durable occurrence. pub const MAX_EVIDENCE_EXCERPT_BYTES: usize = 4 * 1024; -/// Failure to construct a knowledge contract value. -#[derive(Debug, Error, Clone, PartialEq, Eq)] -pub enum KnowledgeContractError { - /// A branded knowledge identifier was malformed. - #[error("invalid {kind} identifier")] - InvalidIdentifier { - /// Identifier class being validated. - kind: &'static str, - }, - /// A digest was not lowercase hexadecimal with the required length. - #[error("{field} must be exactly {length} lowercase hexadecimal characters")] - InvalidDigest { - /// Field being validated. - field: &'static str, - /// Required character length. - length: usize, - }, - /// A half-open byte range was reversed. - #[error("byte range start {start} exceeds end {end}")] - ReversedRange { - /// Inclusive range start. - start: u64, - /// Exclusive range end. - end: u64, - }, - /// A line range was zero-based or reversed. - #[error("line range must be one-based and ordered; received {start}..={end}")] - InvalidLineRange { - /// Inclusive first line. - start: u32, - /// Inclusive last line. - end: u32, - }, - /// An evidence excerpt exceeded its durable bound. - #[error("evidence excerpt is {actual} bytes; maximum is {maximum}")] - ExcerptTooLarge { - /// Supplied byte length. - actual: usize, - /// Maximum byte length. - maximum: usize, - }, -} - macro_rules! knowledge_id { ($name:ident, $prefix:literal, $doc:literal) => { #[doc = $doc] @@ -644,22 +603,6 @@ pub struct CoverageGap { pub kind: CoverageGapKind, } -/// Coverage carried by every knowledge result. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct KnowledgeCoverage { - /// Strongest analysis quality actually used. - pub analysis_quality: AnalysisQuality, - /// Number of source items examined. - pub sources_examined: u64, - /// Number of evidence occurrences returned or recorded. - pub occurrences: u64, - /// Whether the operation stopped at a configured bound. - pub truncated: bool, - /// Typed omissions; empty only when no known gap exists. - pub gaps: Vec, -} - /// Exact bounded evidence occurrence suitable for durable explanation. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -762,106 +705,4 @@ fn validate_lower_hex( } #[cfg(test)] -mod tests { - use super::*; - - fn descriptor() -> SnapshotDescriptor { - SnapshotDescriptor { - workspace_id: WorkspaceId::new("demo").expect("workspace"), - journal_id: JournalId::new("canonical").expect("journal"), - frontier: 42, - anchor_event_id_hex: "ab".repeat(16), - source_snapshot_id: Some(SourceSnapshotId::derive("source-state")), - } - } - - #[test] - fn snapshot_token_is_deterministic_and_sensitive_to_every_coordinate() { - let first = descriptor(); - let mut changed = first.clone(); - changed.frontier += 1; - assert_eq!( - SnapshotToken::for_descriptor(&first), - SnapshotToken::for_descriptor(&first) - ); - assert_ne!( - SnapshotToken::for_descriptor(&first), - SnapshotToken::for_descriptor(&changed) - ); - let token = SnapshotToken::for_descriptor(&first); - assert_eq!( - SnapshotToken::resolve_for_journal( - token.as_str(), - &first.workspace_id, - &first.journal_id, - ), - Ok(first) - ); - } - - #[test] - fn snapshot_token_rejects_tampering_and_cross_workspace_reuse() { - let descriptor = descriptor(); - let token = SnapshotToken::for_descriptor(&descriptor); - let changed = token.as_str().replacen(".42.", ".41.", 1); - assert!(SnapshotToken::resolve_for_journal( - &changed, - &descriptor.workspace_id, - &descriptor.journal_id, - ) - .is_err()); - assert!(SnapshotToken::resolve_for_journal( - token.as_str(), - &WorkspaceId::new("other").expect("workspace"), - &descriptor.journal_id, - ) - .is_err()); - assert!(SnapshotToken::resolve_for_journal( - token.as_str(), - &descriptor.workspace_id, - &JournalId::new("replica").expect("journal"), - ) - .is_err()); - } - - #[test] - fn git_object_ids_enforce_algorithm_length_and_lowercase_hex() { - assert!(GitObjectId::new(GitObjectFormat::Sha1, "a".repeat(40)).is_ok()); - assert!(GitObjectId::new(GitObjectFormat::Sha256, "b".repeat(64)).is_ok()); - assert!(GitObjectId::new(GitObjectFormat::Sha1, "A".repeat(40)).is_err()); - assert!(GitObjectId::new(GitObjectFormat::Sha256, "c".repeat(40)).is_err()); - } - - #[test] - fn evidence_bounds_fail_closed() { - let occurrence = EvidenceOccurrence { - occurrence_id: EvidenceOccurrenceId::derive("occurrence"), - snapshot_id: SourceSnapshotId::derive("snapshot"), - source_kind: EvidenceSourceKind::Markdown, - path: "docs/a.md".to_string(), - byte_range: ByteRange::new(0, 3).expect("range"), - line_range: LineRange::new(1, 1).expect("line range"), - git_blob: None, - source_digest_hex: "d".repeat(64), - excerpt: "abc".to_string(), - analyzer_fingerprint: "markdown:v1".to_string(), - analysis_quality: AnalysisQuality::Syntactic, - }; - assert_eq!(occurrence.validate(), Ok(())); - - let mut too_large = occurrence; - too_large.excerpt = "x".repeat(MAX_EVIDENCE_EXCERPT_BYTES + 1); - assert!(matches!( - too_large.validate(), - Err(KnowledgeContractError::ExcerptTooLarge { .. }) - )); - } - - #[test] - fn temporal_relation_does_not_collapse_concurrency_into_order() { - let encoded = serde_json::to_string(&TemporalRelation::Concurrent).expect("serialize"); - assert_eq!(encoded, "\"concurrent\""); - assert_ne!(TemporalRelation::Concurrent, TemporalRelation::Before); - assert_ne!(TemporalRelation::Concurrent, TemporalRelation::After); - } -} +mod tests; diff --git a/src/knowledge/contracts.rs b/src/knowledge/contracts.rs new file mode 100644 index 0000000..5f4d36b --- /dev/null +++ b/src/knowledge/contracts.rs @@ -0,0 +1,63 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::{AnalysisQuality, CoverageGap}; + +/// Failure to construct a knowledge contract value. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum KnowledgeContractError { + /// A branded knowledge identifier was malformed. + #[error("invalid {kind} identifier")] + InvalidIdentifier { + /// Identifier class being validated. + kind: &'static str, + }, + /// A digest was not lowercase hexadecimal with the required length. + #[error("{field} must be exactly {length} lowercase hexadecimal characters")] + InvalidDigest { + /// Field being validated. + field: &'static str, + /// Required character length. + length: usize, + }, + /// A half-open byte range was reversed. + #[error("byte range start {start} exceeds end {end}")] + ReversedRange { + /// Inclusive range start. + start: u64, + /// Exclusive range end. + end: u64, + }, + /// A line range was zero-based or reversed. + #[error("line range must be one-based and ordered; received {start}..={end}")] + InvalidLineRange { + /// Inclusive first line. + start: u32, + /// Inclusive last line. + end: u32, + }, + /// An evidence excerpt exceeded its durable bound. + #[error("evidence excerpt is {actual} bytes; maximum is {maximum}")] + ExcerptTooLarge { + /// Supplied byte length. + actual: usize, + /// Maximum byte length. + maximum: usize, + }, +} + +/// Coverage carried by every knowledge result. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct KnowledgeCoverage { + /// Strongest analysis quality actually used. + pub analysis_quality: AnalysisQuality, + /// Number of source items examined. + pub sources_examined: u64, + /// Number of evidence occurrences returned or recorded. + pub occurrences: u64, + /// Whether the operation stopped at a configured bound. + pub truncated: bool, + /// Typed omissions; empty only when no known gap exists. + pub gaps: Vec, +} diff --git a/src/knowledge/tests.rs b/src/knowledge/tests.rs new file mode 100644 index 0000000..87a611d --- /dev/null +++ b/src/knowledge/tests.rs @@ -0,0 +1,97 @@ +use super::*; + +fn descriptor() -> SnapshotDescriptor { + SnapshotDescriptor { + workspace_id: WorkspaceId::new("demo").expect("workspace"), + journal_id: JournalId::new("canonical").expect("journal"), + frontier: 42, + anchor_event_id_hex: "ab".repeat(16), + source_snapshot_id: Some(SourceSnapshotId::derive("source-state")), + } +} + +#[test] +fn snapshot_token_is_deterministic_and_sensitive_to_every_coordinate() { + let first = descriptor(); + let mut changed = first.clone(); + changed.frontier += 1; + assert_eq!( + SnapshotToken::for_descriptor(&first), + SnapshotToken::for_descriptor(&first) + ); + assert_ne!( + SnapshotToken::for_descriptor(&first), + SnapshotToken::for_descriptor(&changed) + ); + let token = SnapshotToken::for_descriptor(&first); + assert_eq!( + SnapshotToken::resolve_for_journal(token.as_str(), &first.workspace_id, &first.journal_id,), + Ok(first) + ); +} + +#[test] +fn snapshot_token_rejects_tampering_and_cross_workspace_reuse() { + let descriptor = descriptor(); + let token = SnapshotToken::for_descriptor(&descriptor); + let changed = token.as_str().replacen(".42.", ".41.", 1); + assert!(SnapshotToken::resolve_for_journal( + &changed, + &descriptor.workspace_id, + &descriptor.journal_id, + ) + .is_err()); + assert!(SnapshotToken::resolve_for_journal( + token.as_str(), + &WorkspaceId::new("other").expect("workspace"), + &descriptor.journal_id, + ) + .is_err()); + assert!(SnapshotToken::resolve_for_journal( + token.as_str(), + &descriptor.workspace_id, + &JournalId::new("replica").expect("journal"), + ) + .is_err()); +} + +#[test] +fn git_object_ids_enforce_algorithm_length_and_lowercase_hex() { + assert!(GitObjectId::new(GitObjectFormat::Sha1, "a".repeat(40)).is_ok()); + assert!(GitObjectId::new(GitObjectFormat::Sha256, "b".repeat(64)).is_ok()); + assert!(GitObjectId::new(GitObjectFormat::Sha1, "A".repeat(40)).is_err()); + assert!(GitObjectId::new(GitObjectFormat::Sha256, "c".repeat(40)).is_err()); +} + +#[test] +fn evidence_bounds_fail_closed() { + let occurrence = EvidenceOccurrence { + occurrence_id: EvidenceOccurrenceId::derive("occurrence"), + snapshot_id: SourceSnapshotId::derive("snapshot"), + source_kind: EvidenceSourceKind::Markdown, + path: "docs/a.md".to_string(), + byte_range: ByteRange::new(0, 3).expect("range"), + line_range: LineRange::new(1, 1).expect("line range"), + git_blob: None, + source_digest_hex: "d".repeat(64), + excerpt: "abc".to_string(), + analyzer_fingerprint: "markdown:v1".to_string(), + analysis_quality: AnalysisQuality::Syntactic, + }; + assert_eq!(occurrence.validate(), Ok(())); + + let mut too_large = occurrence; + too_large.excerpt = "x".repeat(MAX_EVIDENCE_EXCERPT_BYTES + 1); + assert!(matches!( + too_large.validate(), + Err(KnowledgeContractError::ExcerptTooLarge { .. }) + )); +} + +#[test] +fn temporal_relation_does_not_collapse_concurrency_into_order() { + let encoded = serde_json::to_string(&TemporalRelation::Concurrent).expect("serialize"); + assert_eq!(encoded, "\"concurrent\""); + assert_ne!(TemporalRelation::Concurrent, TemporalRelation::Before); + assert_ne!(TemporalRelation::Concurrent, TemporalRelation::After); +} diff --git a/src/ops/agent.rs b/src/ops/agent.rs index 40d338a..6058b39 100644 --- a/src/ops/agent.rs +++ b/src/ops/agent.rs @@ -10,11 +10,8 @@ //! //! A failed session-end leaves the lane archive intact and may be retried; no //! transcript restore path is needed. -#![expect( - missing_docs, - reason = "syncbat::operation generates public registration shims without doc injection hooks" -)] +mod chat; use batpak::event::EventPayload; use batpak::id::{EntityIdType, IdempotencyKey}; use batpak::store::{AppendOptions, AppendPositionHint}; @@ -23,7 +20,6 @@ use std::path::{Path, PathBuf}; use syncbat::{CoreBuilder, HandlerResult, OperationRegisterItem}; use crate::claims::session_log::TurnEntry; -use crate::claims::workspace::WorkspaceView; use crate::error::TexoError; use crate::events::coordinate::{coordinate_for_session, entity_for_session, session_lane}; use crate::events::payloads::{ @@ -33,9 +29,12 @@ use crate::journal_store::JournalStore; use crate::ops::env::{self, ReceiptNote}; use crate::ops::handlers::{ append_json, assemble_current_view, infer_supersessions, op_runtime, parse_input, plan_sources, - run_op, run_relate_pass, take_receipts, workspace_temporal_policy, + run_op, run_relate_pass, take_receipts, workspace_temporal_policy, ExplicitSupersessionOutcome, + SourcePlan, }; +use chat::{complete_chat, memory_snapshot, model_role_enabled}; + /// Directory under the workspace root where session transcripts land. pub const SESSIONS_DIR: &str = "sessions"; /// Maximum accepted session id length. @@ -174,8 +173,6 @@ pub fn render_transcript(session_id: &str, turns: &[TurnEntry], include_assistan #[syncbat::operation( descriptor = AGENT_CHAT, - register = register_agent_chat, - register_item = agent_chat_item, name = "texo.agent.chat", effect = Persist, input_schema = "texo.agent.chat.input.v2", @@ -223,11 +220,9 @@ fn agent_chat(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { let (memory, history) = env::with(|op_env| { let mut cache = op_env.cache.borrow_mut(); - let view = crate::claims::workspace::assemble( - &op_env.store, - &op_env.workspace_id, - &mut cache, - )?; + let view = crate::ops::env::replay_scope(|| { + crate::claims::workspace::assemble(&op_env.store, &op_env.workspace_id, &mut cache) + })?; let memory = memory_snapshot(&view); let history = read_session_turns(&op_env.store, &input.session_id)?; Ok::<_, TexoError>((memory, history)) @@ -259,8 +254,6 @@ fn agent_chat(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { #[syncbat::operation( descriptor = AGENT_MEMORY, - register = register_agent_memory, - register_item = agent_memory_item, name = "texo.agent.memory", effect = Inspect, input_schema = "texo.agent.memory.input.v2", @@ -282,8 +275,6 @@ fn agent_memory(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { #[syncbat::operation( descriptor = AGENT_SESSION_END, - register = register_agent_session_end, - register_item = agent_session_end_item, name = "texo.agent.session.end", effect = Persist, input_schema = "texo.agent.session.end.input.v2", @@ -294,122 +285,167 @@ fn agent_memory(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { queries_projections = ["texo.workspace.view.v2"] )] #[tracing::instrument(skip_all)] -#[expect( - clippy::too_many_lines, - reason = "session settlement keeps transcript, ingest, and relate ordering in one operation" -)] fn agent_session_end(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { run_op("texo.agent.session.end", || { let input: AgentSessionEndInput = parse_input("texo.agent.session.end", input)?; validate_session_input("texo.agent.session.end", &input.session_id)?; - cx.event_read_handle() - .read_event("evt.e008") - .map_err(|error| op_runtime("texo.agent.session.end", error))?; - cx.projection_read_handle() - .query_projection("texo.workspace.view.v2") - .map_err(|error| op_runtime("texo.agent.session.end", error))?; - let turns = env::with(|op_env| read_session_turns(&op_env.store, &input.session_id))??; - if turns.is_empty() { - return Err(TexoError::MissingEntity { - entity: entity_for_session(&input.session_id), - }); - } - let (root, workspace_id, extractor_cmd) = env::with(|op_env| { - ( - op_env.root.clone(), - op_env.workspace_id.clone(), - op_env.config.extractor_cmd.clone(), - ) - })?; - let doc_path = session_doc_path(&root, &input.session_id); - if let Some(parent) = doc_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write( - &doc_path, - render_transcript(&input.session_id, &turns, false), - )?; - let view = assemble_current_view()?; - let planned = plan_sources( - "texo.agent.session.end", - &root, - &doc_path, - &workspace_id, - input.observed_at_ms, - extractor_cmd.as_deref(), - &view, - )?; - if !planned.skipped.is_empty() { - return Err(TexoError::Source { - path: doc_path.to_string_lossy().to_string(), - detail: "generated session transcript could not be planned".to_string(), - }); - } - let new_claims = planned - .sources - .iter() - .flat_map(|source| source.claims.iter().cloned()) - .collect::>(); - let temporal = workspace_temporal_policy(&view)?; - let supersessions = - infer_supersessions(&view, &new_claims, input.observed_at_ms, &temporal)?; - for source in &planned.sources { - append_json( - "texo.agent.session.end", - cx, - ::KIND, - &source.observed, - )?; - for claim in &source.claims { - append_json( - "texo.agent.session.end", - cx, - ::KIND, - claim, - )?; - } - } - for supersession in &supersessions.applied { - append_json( - "texo.agent.session.end", - cx, - ::KIND, - supersession, - )?; - } + authorize_session_end_reads(cx)?; + let context = prepare_session_end(&input)?; + append_session_ingest(cx, &context.plan, &context.supersessions)?; drop(take_receipts()?); - let relate = if model_role_enabled(crate::gateway::ModelRole::Relate)? { - let out = run_relate_pass("texo.agent.session.end", cx, input.observed_at_ms, false)?; - RelateOutcome::Ran { - supersessions: out.supersessions.len(), - conflicts: out.conflicts.len(), - } - } else { - RelateOutcome::Skipped { - reason: "TEXO_LLM_API_KEY is not set".to_string(), - } - }; + let relate = run_session_relate(cx, input.observed_at_ms)?; Ok(SessionEndReport { session_id: input.session_id, - doc_path: doc_path - .strip_prefix(&root) - .unwrap_or(&doc_path) + doc_path: context + .doc_path + .strip_prefix(&context.root) + .unwrap_or(&context.doc_path) .to_string_lossy() .to_string(), - sources_observed: u32::try_from(planned.sources.len()).unwrap_or(u32::MAX), - claims_recorded: u32::try_from(new_claims.len()).unwrap_or(u32::MAX), - ingest_supersessions: u32::try_from(supersessions.applied.len()).unwrap_or(u32::MAX), - supersessions_held: supersessions.held.len(), - held_supersessions: supersessions.held, + sources_observed: u32::try_from(context.plan.sources.len()).unwrap_or(u32::MAX), + claims_recorded: u32::try_from(context.claim_count).unwrap_or(u32::MAX), + ingest_supersessions: u32::try_from(context.supersessions.applied.len()) + .unwrap_or(u32::MAX), + supersessions_held: context.supersessions.held.len(), + held_supersessions: context.supersessions.held, relate, }) }) } +struct SessionEndContext { + root: PathBuf, + doc_path: PathBuf, + plan: SourcePlan, + claim_count: usize, + supersessions: ExplicitSupersessionOutcome, +} + +fn authorize_session_end_reads(cx: &mut syncbat::Ctx<'_>) -> Result<(), TexoError> { + cx.event_read_handle() + .read_event("evt.e008") + .map_err(|error| op_runtime("texo.agent.session.end", error))?; + cx.projection_read_handle() + .query_projection("texo.workspace.view.v2") + .map_err(|error| op_runtime("texo.agent.session.end", error))?; + Ok(()) +} + +fn prepare_session_end(input: &AgentSessionEndInput) -> Result { + let turns = env::with(|op_env| read_session_turns(&op_env.store, &input.session_id))??; + if turns.is_empty() { + return Err(TexoError::MissingEntity { + entity: entity_for_session(&input.session_id), + }); + } + let (root, workspace_id, extractor_cmd) = env::with(|op_env| { + ( + op_env.root.clone(), + op_env.workspace_id.clone(), + op_env.config.extractor_cmd.clone(), + ) + })?; + let doc_path = write_session_transcript(&root, &input.session_id, &turns)?; + let view = assemble_current_view()?; + let plan = plan_sources( + "texo.agent.session.end", + &root, + &doc_path, + &workspace_id, + input.observed_at_ms, + extractor_cmd.as_deref(), + &view, + )?; + if !plan.skipped.is_empty() { + return Err(TexoError::Source { + path: doc_path.to_string_lossy().to_string(), + detail: "generated session transcript could not be planned".to_string(), + }); + } + let new_claims = plan + .sources + .iter() + .flat_map(|source| source.claims.iter().cloned()) + .collect::>(); + let temporal = workspace_temporal_policy(&view)?; + let supersessions = infer_supersessions(&view, &new_claims, input.observed_at_ms, &temporal)?; + Ok(SessionEndContext { + root, + doc_path, + plan, + claim_count: new_claims.len(), + supersessions, + }) +} + +fn write_session_transcript( + root: &Path, + session_id: &str, + turns: &[TurnEntry], +) -> Result { + let doc_path = session_doc_path(root, session_id); + if let Some(parent) = doc_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&doc_path, render_transcript(session_id, turns, false))?; + Ok(doc_path) +} + +fn append_session_ingest( + cx: &mut syncbat::Ctx<'_>, + plan: &SourcePlan, + supersessions: &ExplicitSupersessionOutcome, +) -> Result<(), TexoError> { + for source in &plan.sources { + append_json( + "texo.agent.session.end", + cx, + ::KIND, + &source.observed, + )?; + for claim in &source.claims { + append_json( + "texo.agent.session.end", + cx, + ::KIND, + claim, + )?; + } + } + for supersession in &supersessions.applied { + append_json( + "texo.agent.session.end", + cx, + ::KIND, + supersession, + )?; + } + Ok(()) +} + +fn run_session_relate( + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, +) -> Result { + if !model_role_enabled(crate::gateway::ModelRole::Relate)? { + return Ok(RelateOutcome::Skipped { + reason: "TEXO_LLM_API_KEY is not set".to_string(), + }); + } + let out = run_relate_pass( + "texo.agent.session.end", + cx, + observed_at_ms, + crate::ops::handlers::RelatePassOptions::best_effort(), + )?; + Ok(RelateOutcome::Ran { + supersessions: out.supersessions.len(), + conflicts: out.conflicts.len(), + }) +} + #[syncbat::operation( descriptor = SESSION_EXPORT, - register = register_session_export, - register_item = session_export_item, name = "texo.session.export", effect = Inspect, input_schema = "texo.session.export.input.v2", @@ -442,10 +478,10 @@ fn session_export(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { #[must_use] pub fn catalog() -> Vec { vec![ - agent_chat_item(), - agent_memory_item(), - agent_session_end_item(), - session_export_item(), + OperationRegisterItem::new((*AGENT_CHAT).clone(), agent_chat), + OperationRegisterItem::new((*AGENT_MEMORY).clone(), agent_memory), + OperationRegisterItem::new((*AGENT_SESSION_END).clone(), agent_session_end), + OperationRegisterItem::new((*SESSION_EXPORT).clone(), session_export), ] } @@ -456,10 +492,9 @@ pub fn catalog() -> Vec { /// Returns [`syncbat::BuildError`] if a descriptor or handler cannot be /// registered with the builder. pub fn register_all(builder: &mut CoreBuilder) -> Result<(), syncbat::BuildError> { - register_agent_chat(builder)?; - register_agent_memory(builder)?; - register_agent_session_end(builder)?; - register_session_export(builder)?; + for item in catalog() { + let _builder = builder.register_item(item)?; + } Ok(()) } @@ -686,170 +721,6 @@ fn append_turn_direct( }) } -fn memory_snapshot(view: &WorkspaceView) -> MemorySnapshot { - let current = view - .claims - .iter() - .filter(|claim| claim.card.phase != 2) - .map(|claim| MemoryClaim { - claim_id: claim.card.claim_id.clone(), - text: claim.card.text.clone(), - source_path: claim.card.source_path.clone(), - line: claim.card.line_start, - char_start: claim.card.char_start, - char_end: claim.card.char_end, - }) - .collect::>(); - let stale = view - .claims - .iter() - .filter(|claim| claim.card.phase == 2) - .filter_map(|claim| { - claim.card.superseded_by.as_ref().map(|superseded_by| { - let superseded_by_text = view - .claims - .iter() - .find(|candidate| candidate.card.claim_id == *superseded_by) - .map_or_else(String::new, |candidate| candidate.card.text.clone()); - StaleMemory { - claim_id: claim.card.claim_id.clone(), - text: claim.card.text.clone(), - superseded_by: superseded_by.clone(), - superseded_by_text, - } - }) - }) - .collect::>(); - let conflicts = view - .conflicts - .iter() - .filter(|conflict| conflict.phase == 1) - .map(|conflict| MemoryConflict { - claim_a_text: claim_text(view, &conflict.claim_a), - claim_b_text: claim_text(view, &conflict.claim_b), - reason: conflict.reason.clone(), - }) - .collect::>(); - MemorySnapshot { - workspace_id: view.workspace_id.clone(), - replayed_through_sequence: view.frontier, - current, - stale, - conflicts, - } -} - -fn claim_text(view: &WorkspaceView, claim_id: &str) -> String { - view.claims - .iter() - .find(|claim| claim.card.claim_id == claim_id) - .map_or_else(String::new, |claim| claim.card.text.clone()) -} - -fn model_role_enabled(role: crate::gateway::ModelRole) -> Result { - let resolved = env::with(|op_env| { - crate::gateway::resolve_role( - role, - &crate::gateway::RoleOverrides::default(), - op_env.config.gateway.as_ref(), - ) - })?; - Ok(crate::host::grants_model_capability(Some(resolved.api_key))) -} - -#[cfg(feature = "openrouter")] -fn complete_chat( - memory: &MemorySnapshot, - history: &[TurnEntry], - user_message: &str, -) -> Result { - let role = env::with(|op_env| { - crate::gateway::resolve_role( - crate::gateway::ModelRole::Chat, - &crate::gateway::RoleOverrides::default(), - op_env.config.gateway.as_ref(), - ) - })?; - if !role.is_enabled() { - return Err(TexoError::Model { - detail: "chat is disabled: TEXO_LLM_API_KEY is not set".to_string(), - }); - } - let chat_memory = to_chat_memory(memory); - let chat_history = history - .iter() - .filter_map(|turn| { - let speaker = match turn.speaker.as_str() { - "user" => crate::semantics::chat::Speaker::User, - "assistant" => crate::semantics::chat::Speaker::Assistant, - _ => return None, - }; - Some(crate::semantics::chat::Utterance { - speaker, - text: turn.text.clone(), - }) - }) - .collect::>(); - let system_prompt = crate::semantics::chat::build_system_prompt(&chat_memory); - let body = crate::semantics::chat::build_chat_request( - &role, - &system_prompt, - &chat_history, - user_message, - ); - crate::semantics::chat::complete(&role, &body) -} - -#[cfg(not(feature = "openrouter"))] -fn complete_chat( - _memory: &MemorySnapshot, - _history: &[TurnEntry], - _user_message: &str, -) -> Result { - Err(TexoError::Model { - detail: "chat is disabled: openrouter feature is disabled".to_string(), - }) -} - -#[cfg(feature = "openrouter")] -fn to_chat_memory(memory: &MemorySnapshot) -> crate::semantics::chat::MemorySnapshot { - crate::semantics::chat::MemorySnapshot { - workspace_id: memory.workspace_id.clone(), - replayed_through_sequence: memory.replayed_through_sequence, - current: memory - .current - .iter() - .map(|claim| crate::semantics::chat::MemoryClaim { - claim_id: claim.claim_id.clone(), - text: claim.text.clone(), - source_path: claim.source_path.clone(), - line: claim.line, - char_start: claim.char_start, - char_end: claim.char_end, - }) - .collect(), - stale: memory - .stale - .iter() - .map(|stale| crate::semantics::chat::StaleMemory { - claim_id: stale.claim_id.clone(), - text: stale.text.clone(), - superseded_by: stale.superseded_by.clone(), - superseded_by_text: stale.superseded_by_text.clone(), - }) - .collect(), - conflicts: memory - .conflicts - .iter() - .map(|conflict| crate::semantics::chat::MemoryConflict { - claim_a_text: conflict.claim_a_text.clone(), - claim_b_text: conflict.claim_b_text.clone(), - reason: conflict.reason.clone(), - }) - .collect(), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/ops/agent/chat.rs b/src/ops/agent/chat.rs new file mode 100644 index 0000000..68ee95b --- /dev/null +++ b/src/ops/agent/chat.rs @@ -0,0 +1,170 @@ +//! Agent memory projection and model-bound chat composition. + +use crate::claims::session_log::TurnEntry; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::ops::env; + +use super::{MemoryClaim, MemoryConflict, MemorySnapshot, StaleMemory}; + +pub(super) fn memory_snapshot(view: &WorkspaceView) -> MemorySnapshot { + let current = view + .claims + .iter() + .filter(|claim| claim.card.phase != 2) + .map(|claim| MemoryClaim { + claim_id: claim.card.claim_id.clone(), + text: claim.card.text.clone(), + source_path: claim.card.source_path.clone(), + line: claim.card.line_start, + char_start: claim.card.char_start, + char_end: claim.card.char_end, + }) + .collect::>(); + let stale = view + .claims + .iter() + .filter(|claim| claim.card.phase == 2) + .filter_map(|claim| { + claim.card.superseded_by.as_ref().map(|superseded_by| { + let superseded_by_text = claim_text(view, superseded_by); + StaleMemory { + claim_id: claim.card.claim_id.clone(), + text: claim.card.text.clone(), + superseded_by: superseded_by.clone(), + superseded_by_text, + } + }) + }) + .collect::>(); + let conflicts = view + .conflicts + .iter() + .filter(|conflict| conflict.phase == 1) + .map(|conflict| MemoryConflict { + claim_a_text: claim_text(view, &conflict.claim_a), + claim_b_text: claim_text(view, &conflict.claim_b), + reason: conflict.reason.clone(), + }) + .collect::>(); + MemorySnapshot { + workspace_id: view.workspace_id.clone(), + replayed_through_sequence: view.frontier, + current, + stale, + conflicts, + } +} + +fn claim_text(view: &WorkspaceView, claim_id: &str) -> String { + view.claims + .iter() + .find(|claim| claim.card.claim_id == claim_id) + .map_or_else(String::new, |claim| claim.card.text.clone()) +} + +pub(super) fn model_role_enabled(role: crate::gateway::ModelRole) -> Result { + let resolved = env::with(|op_env| { + crate::gateway::resolve_role( + role, + &crate::gateway::RoleOverrides::default(), + op_env.config.gateway.as_ref(), + ) + })?; + Ok(crate::host::grants_model_capability(Some( + resolved.api_key.as_str(), + ))) +} + +#[cfg(feature = "openrouter")] +pub(super) fn complete_chat( + memory: &MemorySnapshot, + history: &[TurnEntry], + user_message: &str, +) -> Result { + let role = env::with(|op_env| { + crate::gateway::resolve_role( + crate::gateway::ModelRole::Chat, + &crate::gateway::RoleOverrides::default(), + op_env.config.gateway.as_ref(), + ) + })?; + if !role.is_enabled() { + return Err(TexoError::Model { + detail: "chat is disabled: TEXO_LLM_API_KEY is not set".to_string(), + }); + } + let chat_memory = to_chat_memory(memory); + let chat_history = history + .iter() + .filter_map(|turn| { + let speaker = match turn.speaker.as_str() { + "user" => crate::semantics::chat::Speaker::User, + "assistant" => crate::semantics::chat::Speaker::Assistant, + _ => return None, + }; + Some(crate::semantics::chat::Utterance { + speaker, + text: turn.text.clone(), + }) + }) + .collect::>(); + let system_prompt = crate::semantics::chat::build_system_prompt(&chat_memory); + let body = crate::semantics::chat::build_chat_request( + &role, + &system_prompt, + &chat_history, + user_message, + ); + crate::semantics::chat::complete(&role, &body) +} + +#[cfg(not(feature = "openrouter"))] +pub(super) fn complete_chat( + _memory: &MemorySnapshot, + _history: &[TurnEntry], + _user_message: &str, +) -> Result { + Err(TexoError::Model { + detail: "chat is disabled: openrouter feature is disabled".to_string(), + }) +} + +#[cfg(feature = "openrouter")] +fn to_chat_memory(memory: &MemorySnapshot) -> crate::semantics::chat::MemorySnapshot { + crate::semantics::chat::MemorySnapshot { + workspace_id: memory.workspace_id.clone(), + replayed_through_sequence: memory.replayed_through_sequence, + current: memory + .current + .iter() + .map(|claim| crate::semantics::chat::MemoryClaim { + claim_id: claim.claim_id.clone(), + text: claim.text.clone(), + source_path: claim.source_path.clone(), + line: claim.line, + char_start: claim.char_start, + char_end: claim.char_end, + }) + .collect(), + stale: memory + .stale + .iter() + .map(|stale| crate::semantics::chat::StaleMemory { + claim_id: stale.claim_id.clone(), + text: stale.text.clone(), + superseded_by: stale.superseded_by.clone(), + superseded_by_text: stale.superseded_by_text.clone(), + }) + .collect(), + conflicts: memory + .conflicts + .iter() + .map(|conflict| crate::semantics::chat::MemoryConflict { + claim_a_text: conflict.claim_a_text.clone(), + claim_b_text: conflict.claim_b_text.clone(), + reason: conflict.reason.clone(), + }) + .collect(), + } +} diff --git a/src/ops/backend.rs b/src/ops/backend.rs index b49f72e..992ab9e 100644 --- a/src/ops/backend.rs +++ b/src/ops/backend.rs @@ -1,34 +1,18 @@ //! Texo syncbat effect backend. -use batpak::event::{EventKind, EventPayload}; -use batpak::id::{CausationId, CorrelationId, EntityIdType, IdempotencyKey}; -use batpak::store::{AppendOptions, AppendPositionHint, AppendReceipt}; +use batpak::event::EventKind; +use batpak::id::EntityIdType as _; +use batpak::store::AppendReceipt; use syncbat::{EffectBackend, EffectError}; use crate::error::TexoError; -use crate::events::coordinate::{ - coordinate_for_claim, coordinate_for_code_index, coordinate_for_conflict, - coordinate_for_evidence, coordinate_for_onboarding_projection, coordinate_for_relation_pair, - coordinate_for_session, coordinate_for_source, coordinate_for_source_relation, - coordinate_for_source_snapshot, coordinate_for_workspace_meta, entity_for_evidence, - entity_for_session, entity_for_source_snapshot, scope_for_workspace, session_lane, -}; -use crate::events::ids::relation_pair_id; -use crate::events::machines::{ - ignore_conflict, open_conflict, record_claim, resolve_conflict, supersede_claim, -}; -use crate::events::payloads::{ - ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, CodeIndexRecordedV1, - ConflictOpenedV2, ConflictResolvedV2, EvidenceOccurrenceRecordedV1, - EvidenceReconciliationAcceptedV1, OnboardingCompiledV2, RelationDeferredV1, RelationJudgedV1, - SessionTurnV1, SourceObservedV2, SourceSnapshotRecordedV1, SourceSnapshotRelationV1, - WorkspaceInitializedV2, -}; +use crate::events::coordinate::scope_for_workspace; use crate::ops::env::{self, OpEnv, ReceiptNote}; -/// Runtime-owned durable effect backend for texo operations. -#[derive(Default)] -pub struct TexoEffectBackend; +mod effect; +mod policy; + +pub use effect::TexoEffectBackend; impl EffectBackend for TexoEffectBackend { fn append_event(&mut self, kind: EventKind, payload: &[u8]) -> Result<(), EffectError> { @@ -59,400 +43,13 @@ fn with_env_result(f: impl FnOnce(&OpEnv) -> Result) -> Result< .map_err(|error| effect_error(&error)) } -#[expect( - clippy::too_many_lines, - reason = "single domain append chokepoint owns all typed decoding, coordinates, and idempotency keys" -)] fn append_domain_event( op_env: &OpEnv, kind: EventKind, payload_bytes: &[u8], ) -> Result<(), TexoError> { - if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_claim(&payload.workspace_id, &payload.claim_id)?; - let key = IdempotencyKey::for_operation( - "texo.claim.recorded.v2", - &[&payload.workspace_id, &payload.claim_id], - ); - let payload = record_claim(payload).into_payload(); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_claim(&payload.workspace_id, &payload.old_claim_id)?; - let key = IdempotencyKey::for_operation( - "texo.claim.supersede.v2", - &[ - &payload.workspace_id, - &payload.old_claim_id, - &payload.new_claim_id, - ], - ); - let payload = supersede_claim(payload).into_payload(); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_conflict(&payload.workspace_id, &payload.conflict_id)?; - let key = IdempotencyKey::for_operation( - "texo.conflict.open.v2", - &[&payload.workspace_id, &payload.conflict_id], - ); - let payload = open_conflict(payload).into_payload(); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_conflict(&payload.workspace_id, &payload.conflict_id)?; - let key = IdempotencyKey::for_operation( - "texo.conflict.resolve.v2", - &[&payload.workspace_id, &payload.conflict_id], - ); - let receipt = match payload.resolution.as_str() { - "resolved" => { - let payload = resolve_conflict(payload).into_payload(); - op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )? - } - "ignored" => { - let payload = ignore_conflict(payload).into_payload(); - op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )? - } - other => { - return Err(TexoError::StatusParse { - value: other.to_string(), - }); - } - }; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_source(&payload.workspace_id, &payload.source_id)?; - let key = IdempotencyKey::for_operation( - "texo.source.observed.v2", - &[&payload.workspace_id, &payload.body_hash_hex], - ); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_onboarding_projection(&payload.workspace_id)?; - let receipt = op_env.store.append_typed(&coordinate, &payload)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_workspace_meta(&payload.workspace_id)?; - let key = IdempotencyKey::for_operation( - "texo.workspace.initialized.v2", - &[&payload.workspace_id, &payload.config_digest_hex], - ); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let pair_id = relation_pair_id( - &payload.workspace_id, - &payload.older_claim, - &payload.newer_claim, - ); - let coordinate = - coordinate_for_relation_pair(payload.workspace_id.as_str(), pair_id.as_str())?; - let key = IdempotencyKey::for_operation( - "texo.relation.judged.v1", - &[ - payload.workspace_id.as_str(), - payload.older_claim.as_str(), - payload.newer_claim.as_str(), - &payload.judge_fingerprint, - ], - ); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let pair_id = relation_pair_id( - &payload.workspace_id, - &payload.older_claim, - &payload.newer_claim, - ); - let coordinate = - coordinate_for_relation_pair(payload.workspace_id.as_str(), pair_id.as_str())?; - let receipt = op_env.store.append_typed(&coordinate, &payload)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_source_snapshot( - payload.workspace_id.as_str(), - payload.snapshot_id.as_str(), - )?; - let key = IdempotencyKey::for_operation( - "texo.source.snapshot.recorded.v1", - &[payload.workspace_id.as_str(), payload.snapshot_id.as_str()], - ); - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new().with_idempotency(key), - )?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - payload - .occurrence - .validate() - .map_err(|error| TexoError::OpInput { - // This kind is appended from both knowledge.index and - // knowledge.reconcile, so use the generic dispatcher label the - // other backend-level errors here use rather than one op name. - op: "texo.effect.append".to_string(), - detail: error.to_string(), - })?; - let coordinate = coordinate_for_evidence( - payload.workspace_id.as_str(), - payload.occurrence.occurrence_id.as_str(), - )?; - let key = IdempotencyKey::for_operation( - "texo.evidence.occurrence.recorded.v1", - &[ - payload.workspace_id.as_str(), - payload.occurrence.occurrence_id.as_str(), - ], - ); - let options = - code_index_append_options(op_env, key, payload.occurrence.snapshot_id.as_str()); - let receipt = op_env - .store - .append_typed_with_options(&coordinate, &payload, options)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = - coordinate_for_claim(payload.workspace_id.as_str(), payload.claim_id.as_str())?; - let key = IdempotencyKey::for_operation( - "texo.evidence.reconciliation.accepted.v1", - &[ - payload.workspace_id.as_str(), - payload.claim_id.as_str(), - payload.occurrence_id.as_str(), - &payload.judge_fingerprint, - &payload.policy_version, - ], - ); - let options = evidence_chain_options( - op_env, - key, - payload.occurrence_id.as_str(), - ::KIND, - )?; - let receipt = op_env - .store - .append_typed_with_options(&coordinate, &payload, options)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = - coordinate_for_claim(payload.workspace_id.as_str(), payload.claim_id.as_str())?; - let key = IdempotencyKey::for_operation( - "texo.claim.evidence.linked.v1", - &[ - payload.workspace_id.as_str(), - payload.claim_id.as_str(), - payload.occurrence_id.as_str(), - ], - ); - let options = evidence_chain_options( - op_env, - key, - payload.occurrence_id.as_str(), - ::KIND, - )?; - let receipt = op_env - .store - .append_typed_with_options(&coordinate, &payload, options)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = - coordinate_for_code_index(payload.workspace_id.as_str(), payload.index_id.as_str())?; - let key = IdempotencyKey::for_operation( - "texo.code.index.recorded.v1", - &[payload.workspace_id.as_str(), payload.index_id.as_str()], - ); - let options = code_index_append_options(op_env, key, payload.snapshot_id.as_str()); - let receipt = op_env - .store - .append_typed_with_options(&coordinate, &payload, options)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_source_relation( - payload.workspace_id.as_str(), - payload.left_snapshot_id.as_str(), - payload.right_snapshot_id.as_str(), - )?; - let key = IdempotencyKey::for_operation( - "texo.source.snapshot.relation.v1", - &[ - payload.workspace_id.as_str(), - payload.left_snapshot_id.as_str(), - payload.right_snapshot_id.as_str(), - ], - ); - let options = code_index_append_options(op_env, key, payload.right_snapshot_id.as_str()); - let receipt = op_env - .store - .append_typed_with_options(&coordinate, &payload, options)?; - verify_and_note(op_env, kind, &receipt) - } else if kind == ::KIND { - let payload = decode::(payload_bytes)?; - let coordinate = coordinate_for_session(&payload.workspace_id, &payload.session_id)?; - let entity = entity_for_session(&payload.session_id); - let lane = session_lane(&payload.session_id); - let depth = u32::try_from(op_env.store.stream_lane(&entity, lane).len()).map_err(|_| { - TexoError::OpRuntime { - op: "texo.effect.append".to_string(), - detail: "session lane depth exceeded u32".to_string(), - denied: false, - } - })?; - let hint = if depth == 0 { - AppendPositionHint::branch_root(lane, 0) - } else { - AppendPositionHint::new(lane, depth) - }; - let receipt = op_env.store.append_typed_with_options( - &coordinate, - &payload, - AppendOptions::new() - .with_position_hint(hint) - .with_idempotency(IdempotencyKey::for_operation( - "texo.session.turn.v1", - &[ - &payload.workspace_id, - &payload.session_id, - &payload.turn_no.to_string(), - ], - )), - )?; - verify_and_note(op_env, kind, &receipt) - } else { - Err(TexoError::OpRuntime { - op: "texo.effect.append".to_string(), - detail: format!( - "event kind evt.{:04x} is outside texo domain", - kind.as_raw_u16() - ), - denied: false, - }) - } -} - -fn evidence_chain_options( - op_env: &OpEnv, - key: IdempotencyKey, - occurrence_id: &str, - preferred_cause_kind: EventKind, -) -> Result { - let occurrence_entry = op_env - .store - .by_entity(&entity_for_evidence(occurrence_id)) - .into_iter() - .find(|entry| entry.event_kind() == ::KIND) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity_for_evidence(occurrence_id), - })?; - let raw = op_env.store.read_raw(occurrence_entry.event_id())?; - let occurrence = batpak::encoding::from_bytes::( - &raw.event.payload, - ) - .map_err(|error| TexoError::Decode { - entity: entity_for_evidence(occurrence_id), - detail: error.to_string(), - })?; - let root = op_env - .store - .by_entity(&entity_for_source_snapshot( - occurrence.occurrence.snapshot_id.as_str(), - )) - .into_iter() - .find(|entry| entry.event_kind() == ::KIND) - .map(|entry| entry.event_id().as_u128()); - let receipts = op_env.receipts.borrow(); - let cause = receipts - .iter() - .rev() - .find(|receipt| receipt.kind_bits == preferred_cause_kind.as_raw_u16()) - .and_then(|receipt| u128::from_str_radix(&receipt.event_id_hex, 16).ok()) - .unwrap_or_else(|| occurrence_entry.event_id().as_u128()); - let mut options = AppendOptions::new().with_idempotency(key); - if let Some(root) = root { - options = options.with_correlation(CorrelationId::from(root)); - } - options = options.with_causation(CausationId::from(cause)); - Ok(options) -} - -fn code_index_append_options( - op_env: &OpEnv, - key: IdempotencyKey, - snapshot_id: &str, -) -> AppendOptions { - let root = op_env - .store - .by_entity(&entity_for_source_snapshot(snapshot_id)) - .into_iter() - .find(|entry| entry.event_kind() == ::KIND) - .map(|entry| entry.event_id().as_u128()); - let mut options = AppendOptions::new().with_idempotency(key); - if let Some(root) = root { - options = options - .with_correlation(CorrelationId::from(root)) - .with_causation(CausationId::from(root)); - } - options -} - -fn decode(payload_bytes: &[u8]) -> Result -where - T: serde::de::DeserializeOwned, -{ - batpak::canonical::from_bytes(payload_bytes).map_err(|error| TexoError::OpRuntime { - op: "texo.effect.append".to_string(), - detail: format!("canonical effect payload decoding failed: {error}"), - denied: false, - }) + let receipt = policy::append(op_env, kind, payload_bytes)?; + verify_and_note(op_env, kind, &receipt) } fn verify_and_note( @@ -500,15 +97,21 @@ mod tests { use std::cell::RefCell; use std::sync::Arc; - use batpak::store::{Store, StoreConfig}; + use batpak::event::EventPayload; + use batpak::id::{EntityIdType as _, IdempotencyKey}; + use batpak::store::{AppendOptions, Freshness, Store, StoreConfig}; use super::*; + use crate::claims::campaign::CampaignCard; use crate::claims::workspace::WorkspaceCache; use crate::config::WorkspaceConfig; - use crate::events::coordinate::coordinate_for_claim; - use crate::events::ids::{ClaimId, WorkspaceId}; + use crate::events::coordinate::{coordinate_for_claim, entity_for_relation_campaign}; + use crate::events::ids::{relation_pair_id, ClaimId, WorkspaceId}; use crate::events::machines::record_claim; - use crate::relate::settlement::SettledRelation; + use crate::events::payloads::{ + ClaimRecordedV2, RelationCampaignCheckpointV1, RelationJudgedV1, SourceObservedV2, + }; + use crate::relate::settlement::{CampaignPhase, SettledRelation}; fn test_host_interface() -> crate::host::HostInterface { crate::host::HostInterface { @@ -530,13 +133,9 @@ mod tests { .1 } - #[test] - fn keyed_source_retry_returns_original_event_and_appends_once() { - let root = tempfile::tempdir().expect("tempdir"); - let store = - Arc::new(Store::open(StoreConfig::new(root.path().join("store"))).expect("store")); - let env = OpEnv { - store: crate::journal_store::JournalStore::writable(Arc::clone(&store)), + fn test_env(root: &tempfile::TempDir, store: Arc) -> OpEnv { + OpEnv { + store: crate::journal_store::JournalStore::writable(store), workspace_id: "demo".to_string(), root: root.path().to_path_buf(), config: WorkspaceConfig::demo(), @@ -545,7 +144,39 @@ mod tests { observed_at_ms: 1, host_interface: test_host_interface(), journal: test_journal(), - }; + } + } + + fn campaign_checkpoint( + phase: CampaignPhase, + observed_at_ms: u64, + ) -> RelationCampaignCheckpointV1 { + RelationCampaignCheckpointV1 { + workspace_id: WorkspaceId::try_from("demo").expect("workspace"), + evaluated_basis_digest_hex: "a".repeat(64), + result_basis_digest_hex: "a".repeat(64), + candidate_policy_digest_hex: "b".repeat(64), + phase, + observed_at_ms, + } + } + + fn append_campaign_checkpoint(env: &OpEnv, payload: &RelationCampaignCheckpointV1) { + let bytes = batpak::canonical::to_bytes(payload).expect("canonical payload"); + append_domain_event( + env, + ::KIND, + &bytes, + ) + .expect("append campaign checkpoint"); + } + + #[test] + fn keyed_source_retry_returns_original_event_and_appends_once() { + let root = tempfile::tempdir().expect("tempdir"); + let store = + Arc::new(Store::open(StoreConfig::new(root.path().join("store"))).expect("store")); + let env = test_env(&root, Arc::clone(&store)); let payload = SourceObservedV2 { source_id: "src_aaaaaaaaaaaa".to_string(), workspace_id: "demo".to_string(), @@ -574,17 +205,7 @@ mod tests { let root = tempfile::tempdir().expect("tempdir"); let store = Arc::new(Store::open(StoreConfig::new(root.path().join("store"))).expect("store")); - let env = OpEnv { - store: crate::journal_store::JournalStore::writable(Arc::clone(&store)), - workspace_id: "demo".to_string(), - root: root.path().to_path_buf(), - config: WorkspaceConfig::demo(), - cache: RefCell::new(WorkspaceCache::default()), - receipts: RefCell::new(Vec::new()), - observed_at_ms: 1, - host_interface: test_host_interface(), - journal: test_journal(), - }; + let env = test_env(&root, Arc::clone(&store)); let payload = RelationJudgedV1 { workspace_id: WorkspaceId::try_from("demo").expect("workspace"), older_claim: ClaimId::try_from("claim_aaaaaaaaaaaa").expect("older"), @@ -599,13 +220,14 @@ mod tests { append_domain_event(&env, ::KIND, &bytes).expect("first"); append_domain_event(&env, ::KIND, &bytes).expect("retry"); + let payload_identity = blake3::hash(&bytes).to_hex().to_string(); let expected = IdempotencyKey::for_operation( - "texo.relation.judged.v1", + "texo.relation.judged.v1.payload", &[ "demo", "claim_aaaaaaaaaaaa", "claim_bbbbbbbbbbbb", - "openrouter:model|relation-v2", + payload_identity.as_str(), ], ); let receipts = env.receipts.borrow(); @@ -623,6 +245,47 @@ mod tests { assert_eq!(store.by_entity(&format!("relation:{pair_id}")).len(), 1); } + #[test] + fn complete_checkpoint_after_partial_is_a_new_idempotent_transition() { + let root = tempfile::tempdir().expect("tempdir"); + let store = + Arc::new(Store::open(StoreConfig::new(root.path().join("store"))).expect("store")); + let env = test_env(&root, Arc::clone(&store)); + let complete = campaign_checkpoint(CampaignPhase::Complete, 1); + let partial = campaign_checkpoint( + CampaignPhase::Partial { + next_candidate_cursor: 7, + }, + 2, + ); + + append_campaign_checkpoint(&env, &complete); + append_campaign_checkpoint(&env, &complete); + append_campaign_checkpoint(&env, &partial); + append_campaign_checkpoint(&env, &complete); + append_campaign_checkpoint(&env, &complete); + + let entity = entity_for_relation_campaign("demo"); + let entries = store.by_entity(&entity); + assert_eq!(entries.len(), 3); + let receipts = env.receipts.borrow(); + assert_eq!(receipts[0], receipts[1]); + assert_ne!(receipts[0], receipts[3]); + assert_eq!(receipts[3], receipts[4]); + let resumed = store + .read_raw(entries[2].event_id()) + .expect("resumed checkpoint"); + assert_eq!( + resumed.event.header.causation_id.map(|id| id.as_u128()), + Some(entries[1].event_id().as_u128()) + ); + let card = store + .project::(&entity, &Freshness::Consistent) + .expect("campaign projection") + .expect("campaign card"); + assert_eq!(card.latest, Some(complete)); + } + #[test] fn typed_options_transition_preserves_payload_bytes() { let left_dir = tempfile::tempdir().expect("left tempdir"); diff --git a/src/ops/backend/effect.rs b/src/ops/backend/effect.rs new file mode 100644 index 0000000..89d4c1a --- /dev/null +++ b/src/ops/backend/effect.rs @@ -0,0 +1,3 @@ +/// Runtime-owned durable effect backend for texo operations. +#[derive(Default)] +pub struct TexoEffectBackend; diff --git a/src/ops/backend/policy.rs b/src/ops/backend/policy.rs new file mode 100644 index 0000000..fa53c9c --- /dev/null +++ b/src/ops/backend/policy.rs @@ -0,0 +1,608 @@ +//! Typed append policies for Texo domain events. + +use batpak::event::{EventKind, EventPayload}; +use batpak::id::{CausationId, CorrelationId, EntityIdType, IdempotencyKey}; +use batpak::store::{AppendOptions, AppendPositionHint, AppendReceipt}; + +use crate::error::TexoError; +use crate::events::coordinate::{ + coordinate_for_claim, coordinate_for_code_index, coordinate_for_conflict, + coordinate_for_evidence, coordinate_for_onboarding_projection, + coordinate_for_relation_campaign, coordinate_for_relation_pair, coordinate_for_session, + coordinate_for_source, coordinate_for_source_relation, coordinate_for_source_snapshot, + coordinate_for_workspace_meta, entity_for_evidence, entity_for_relation_campaign, + entity_for_session, entity_for_source_snapshot, session_lane, +}; +use crate::events::ids::relation_pair_id; +use crate::events::machines::{ + ignore_conflict, open_conflict, record_claim, resolve_conflict, supersede_claim, +}; +use crate::events::payloads::{ + ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, CodeIndexRecordedV1, + ConflictOpenedV2, ConflictResolvedV2, EvidenceOccurrenceRecordedV1, + EvidenceReconciliationAcceptedV1, OnboardingCompiledV2, RelationCampaignCheckpointV1, + RelationDeferredV1, RelationJudgedV1, SessionTurnV1, SourceObservedV2, + SourceSnapshotRecordedV1, SourceSnapshotRelationV1, WorkspaceInitializedV2, +}; +use crate::ops::env::OpEnv; +use crate::relate::settlement::CampaignPhase; + +type AppendFn = fn(&OpEnv, &[u8]) -> Result; + +struct AppendPolicy { + kind: EventKind, + append: AppendFn, +} + +const APPEND_POLICIES: &[AppendPolicy] = &[ + policy::(append_claim_recorded), + policy::(append_claim_superseded), + policy::(append_conflict_opened), + policy::(append_conflict_resolved), + policy::(append_source_observed), + policy::(append_onboarding_compiled), + policy::(append_workspace_initialized), + policy::(append_relation_judged), + policy::(append_relation_deferred), + policy::(append_source_snapshot_recorded), + policy::(append_evidence_occurrence_recorded), + policy::(append_evidence_reconciliation_accepted), + policy::(append_claim_evidence_linked), + policy::(append_code_index_recorded), + policy::(append_source_snapshot_relation), + policy::(append_session_turn), + policy::(append_relation_campaign_checkpoint), +]; + +const fn policy(append: AppendFn) -> AppendPolicy { + AppendPolicy { + kind: T::KIND, + append, + } +} + +pub(super) fn append( + op_env: &OpEnv, + kind: EventKind, + payload_bytes: &[u8], +) -> Result { + let selected = APPEND_POLICIES.iter().find(|policy| policy.kind == kind); + selected.map_or_else( + || { + Err(TexoError::OpRuntime { + op: "texo.effect.append".to_string(), + detail: format!( + "event kind evt.{:04x} is outside texo domain", + kind.as_raw_u16() + ), + denied: false, + }) + }, + |policy| (policy.append)(op_env, payload_bytes), + ) +} + +fn append_claim_recorded(op_env: &OpEnv, payload_bytes: &[u8]) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_claim(&payload.workspace_id, &payload.claim_id)?; + let key = IdempotencyKey::for_operation( + "texo.claim.recorded.v2", + &[&payload.workspace_id, &payload.claim_id], + ); + let payload = record_claim(payload).into_payload(); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_claim_superseded( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_claim(&payload.workspace_id, &payload.old_claim_id)?; + let key = IdempotencyKey::for_operation( + "texo.claim.supersede.v2", + &[ + &payload.workspace_id, + &payload.old_claim_id, + &payload.new_claim_id, + ], + ); + let payload = supersede_claim(payload).into_payload(); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_conflict_opened( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_conflict(&payload.workspace_id, &payload.conflict_id)?; + let key = IdempotencyKey::for_operation( + "texo.conflict.open.v2", + &[&payload.workspace_id, &payload.conflict_id], + ); + let payload = open_conflict(payload).into_payload(); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_conflict_resolved( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_conflict(&payload.workspace_id, &payload.conflict_id)?; + let key = IdempotencyKey::for_operation( + "texo.conflict.resolve.v2", + &[&payload.workspace_id, &payload.conflict_id], + ); + match payload.resolution.as_str() { + "resolved" => { + let payload = resolve_conflict(payload).into_payload(); + append_with_key(op_env, &coordinate, &payload, key) + } + "ignored" => { + let payload = ignore_conflict(payload).into_payload(); + append_with_key(op_env, &coordinate, &payload, key) + } + other => Err(TexoError::StatusParse { + value: other.to_string(), + }), + } +} + +fn append_source_observed( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_source(&payload.workspace_id, &payload.source_id)?; + let key = IdempotencyKey::for_operation( + "texo.source.observed.v2", + &[&payload.workspace_id, &payload.body_hash_hex], + ); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_onboarding_compiled( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_onboarding_projection(&payload.workspace_id)?; + op_env + .store + .append_typed(&coordinate, &payload) + .map_err(Into::into) +} + +fn append_workspace_initialized( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_workspace_meta(&payload.workspace_id)?; + let key = IdempotencyKey::for_operation( + "texo.workspace.initialized.v2", + &[&payload.workspace_id, &payload.config_digest_hex], + ); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_relation_judged( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let pair_id = relation_pair_id( + &payload.workspace_id, + &payload.older_claim, + &payload.newer_claim, + ); + let coordinate = coordinate_for_relation_pair(payload.workspace_id.as_str(), pair_id.as_str())?; + let payload_identity = canonical_payload_identity(&payload)?; + let key = IdempotencyKey::for_operation( + "texo.relation.judged.v1.payload", + &[ + payload.workspace_id.as_str(), + payload.older_claim.as_str(), + payload.newer_claim.as_str(), + payload_identity.as_str(), + ], + ); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_relation_deferred( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let pair_id = relation_pair_id( + &payload.workspace_id, + &payload.older_claim, + &payload.newer_claim, + ); + let coordinate = coordinate_for_relation_pair(payload.workspace_id.as_str(), pair_id.as_str())?; + op_env + .store + .append_typed(&coordinate, &payload) + .map_err(Into::into) +} + +fn append_source_snapshot_recorded( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_source_snapshot( + payload.workspace_id.as_str(), + payload.snapshot_id.as_str(), + )?; + let key = IdempotencyKey::for_operation( + "texo.source.snapshot.recorded.v1", + &[payload.workspace_id.as_str(), payload.snapshot_id.as_str()], + ); + append_with_key(op_env, &coordinate, &payload, key) +} + +fn append_evidence_occurrence_recorded( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + payload + .occurrence + .validate() + .map_err(|error| TexoError::OpInput { + op: "texo.effect.append".to_string(), + detail: error.to_string(), + })?; + let coordinate = coordinate_for_evidence( + payload.workspace_id.as_str(), + payload.occurrence.occurrence_id.as_str(), + )?; + let key = IdempotencyKey::for_operation( + "texo.evidence.occurrence.recorded.v1", + &[ + payload.workspace_id.as_str(), + payload.occurrence.occurrence_id.as_str(), + ], + ); + let options = code_index_append_options(op_env, key, payload.occurrence.snapshot_id.as_str()); + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_evidence_reconciliation_accepted( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = + coordinate_for_claim(payload.workspace_id.as_str(), payload.claim_id.as_str())?; + let key = IdempotencyKey::for_operation( + "texo.evidence.reconciliation.accepted.v1", + &[ + payload.workspace_id.as_str(), + payload.claim_id.as_str(), + payload.occurrence_id.as_str(), + &payload.judge_fingerprint, + &payload.policy_version, + ], + ); + let options = evidence_chain_options( + op_env, + key, + payload.occurrence_id.as_str(), + ::KIND, + )?; + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_claim_evidence_linked( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = + coordinate_for_claim(payload.workspace_id.as_str(), payload.claim_id.as_str())?; + let key = IdempotencyKey::for_operation( + "texo.claim.evidence.linked.v1", + &[ + payload.workspace_id.as_str(), + payload.claim_id.as_str(), + payload.occurrence_id.as_str(), + ], + ); + let options = evidence_chain_options( + op_env, + key, + payload.occurrence_id.as_str(), + ::KIND, + )?; + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_code_index_recorded( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = + coordinate_for_code_index(payload.workspace_id.as_str(), payload.index_id.as_str())?; + let key = IdempotencyKey::for_operation( + "texo.code.index.recorded.v1", + &[payload.workspace_id.as_str(), payload.index_id.as_str()], + ); + let options = code_index_append_options(op_env, key, payload.snapshot_id.as_str()); + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_source_snapshot_relation( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_source_relation( + payload.workspace_id.as_str(), + payload.left_snapshot_id.as_str(), + payload.right_snapshot_id.as_str(), + )?; + let key = IdempotencyKey::for_operation( + "texo.source.snapshot.relation.v1", + &[ + payload.workspace_id.as_str(), + payload.left_snapshot_id.as_str(), + payload.right_snapshot_id.as_str(), + ], + ); + let options = code_index_append_options(op_env, key, payload.right_snapshot_id.as_str()); + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_session_turn(op_env: &OpEnv, payload_bytes: &[u8]) -> Result { + let payload = decode::(payload_bytes)?; + let coordinate = coordinate_for_session(&payload.workspace_id, &payload.session_id)?; + let entity = entity_for_session(&payload.session_id); + let lane = session_lane(&payload.session_id); + let depth = u32::try_from(op_env.store.stream_lane(&entity, lane).len()).map_err(|_| { + TexoError::OpRuntime { + op: "texo.effect.append".to_string(), + detail: "session lane depth exceeded u32".to_string(), + denied: false, + } + })?; + let hint = if depth == 0 { + AppendPositionHint::branch_root(lane, 0) + } else { + AppendPositionHint::new(lane, depth) + }; + let key = IdempotencyKey::for_operation( + "texo.session.turn.v1", + &[ + &payload.workspace_id, + &payload.session_id, + &payload.turn_no.to_string(), + ], + ); + let options = AppendOptions::new() + .with_position_hint(hint) + .with_idempotency(key); + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn append_relation_campaign_checkpoint( + op_env: &OpEnv, + payload_bytes: &[u8], +) -> Result { + let payload = decode::(payload_bytes)?; + if payload.workspace_id.as_str() != op_env.workspace_id.as_str() { + return Err(TexoError::OpInput { + op: "texo.effect.append".to_string(), + detail: "relation campaign checkpoint workspace does not match operation workspace" + .to_string(), + }); + } + for (name, digest) in [ + ( + "evaluated basis", + payload.evaluated_basis_digest_hex.as_str(), + ), + ("result basis", payload.result_basis_digest_hex.as_str()), + ( + "candidate policy", + payload.candidate_policy_digest_hex.as_str(), + ), + ] { + if !is_lower_hex_digest(digest) { + return Err(TexoError::OpInput { + op: "texo.effect.append".to_string(), + detail: format!("relation campaign {name} digest must be 64 lowercase hex digits"), + }); + } + } + if matches!(payload.phase, CampaignPhase::Partial { .. }) + && payload.evaluated_basis_digest_hex != payload.result_basis_digest_hex + { + return Err(TexoError::OpInput { + op: "texo.effect.append".to_string(), + detail: "partial relation campaign checkpoint cannot change the claim basis" + .to_string(), + }); + } + let coordinate = coordinate_for_relation_campaign(payload.workspace_id.as_str())?; + let options = campaign_checkpoint_options(op_env, &payload)?; + op_env + .store + .append_typed_with_options(&coordinate, &payload, options) + .map_err(Into::into) +} + +fn campaign_checkpoint_options( + op_env: &OpEnv, + payload: &RelationCampaignCheckpointV1, +) -> Result { + let entity = entity_for_relation_campaign(payload.workspace_id.as_str()); + let latest = op_env + .store + .by_entity(&entity) + .into_iter() + .max_by_key(batpak::store::IndexEntry::global_sequence); + let receipt_predecessor = op_env + .receipts + .borrow() + .last() + .and_then(|receipt| u128::from_str_radix(&receipt.event_id_hex, 16).ok()); + let latest_id = latest.as_ref().map(|entry| entry.event_id().as_u128()); + let retry = receipt_predecessor.is_none_or(|event_id| Some(event_id) == latest_id); + if let Some(entry) = latest.as_ref().filter(|_| retry) { + let raw = op_env.store.read_raw(entry.event_id())?; + let latest = decode::(&raw.event.payload)?; + if latest == *payload { + let key = IdempotencyKey::from(entry.event_id().as_u128()); + return Ok(AppendOptions::new().with_idempotency(key)); + } + } + let predecessor = receipt_predecessor.or(latest_id); + let predecessor_id = + predecessor.map_or_else(|| "root".to_string(), |event_id| format!("{event_id:032x}")); + let payload_identity = canonical_payload_identity(payload)?; + let key = IdempotencyKey::for_operation( + "texo.relation.campaign.checkpoint.transition.v1", + &[ + payload.workspace_id.as_str(), + predecessor_id.as_str(), + payload_identity.as_str(), + ], + ); + let mut options = AppendOptions::new().with_idempotency(key); + if let Some(event_id) = predecessor { + options = options.with_causation(CausationId::from(event_id)); + } + Ok(options) +} + +fn is_lower_hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn append_with_key( + op_env: &OpEnv, + coordinate: &batpak::coordinate::Coordinate, + payload: &T, + key: IdempotencyKey, +) -> Result { + op_env + .store + .append_typed_with_options( + coordinate, + payload, + AppendOptions::new().with_idempotency(key), + ) + .map_err(Into::into) +} + +fn canonical_payload_identity(payload: &T) -> Result { + let bytes = batpak::canonical::to_bytes(payload).map_err(|error| TexoError::OpRuntime { + op: "texo.effect.append".to_string(), + detail: format!("canonical effect payload encoding failed: {error}"), + denied: false, + })?; + Ok(blake3::hash(&bytes).to_hex().to_string()) +} + +fn evidence_chain_options( + op_env: &OpEnv, + key: IdempotencyKey, + occurrence_id: &str, + preferred_cause_kind: EventKind, +) -> Result { + let occurrence_entity = entity_for_evidence(occurrence_id); + let occurrence_entry = op_env + .store + .by_entity(&occurrence_entity) + .into_iter() + .find(|entry| entry.event_kind() == ::KIND) + .ok_or_else(|| TexoError::MissingEntity { + entity: occurrence_entity.clone(), + })?; + let raw = op_env.store.read_raw(occurrence_entry.event_id())?; + let occurrence = batpak::encoding::from_bytes::( + &raw.event.payload, + ) + .map_err(|error| TexoError::Decode { + entity: occurrence_entity, + detail: error.to_string(), + })?; + let root = op_env + .store + .by_entity(&entity_for_source_snapshot( + occurrence.occurrence.snapshot_id.as_str(), + )) + .into_iter() + .find(|entry| entry.event_kind() == ::KIND) + .map(|entry| entry.event_id().as_u128()); + let receipts = op_env.receipts.borrow(); + let cause = receipts + .iter() + .rev() + .find(|receipt| receipt.kind_bits == preferred_cause_kind.as_raw_u16()) + .and_then(|receipt| u128::from_str_radix(&receipt.event_id_hex, 16).ok()) + .unwrap_or_else(|| occurrence_entry.event_id().as_u128()); + let mut options = AppendOptions::new().with_idempotency(key); + if let Some(root) = root { + options = options.with_correlation(CorrelationId::from(root)); + } + Ok(options.with_causation(CausationId::from(cause))) +} + +fn code_index_append_options( + op_env: &OpEnv, + key: IdempotencyKey, + snapshot_id: &str, +) -> AppendOptions { + let root = op_env + .store + .by_entity(&entity_for_source_snapshot(snapshot_id)) + .into_iter() + .find(|entry| entry.event_kind() == ::KIND) + .map(|entry| entry.event_id().as_u128()); + let mut options = AppendOptions::new().with_idempotency(key); + if let Some(root) = root { + options = options + .with_correlation(CorrelationId::from(root)) + .with_causation(CausationId::from(root)); + } + options +} + +fn decode(payload_bytes: &[u8]) -> Result +where + T: serde::de::DeserializeOwned, +{ + batpak::canonical::from_bytes(payload_bytes).map_err(|error| TexoError::OpRuntime { + op: "texo.effect.append".to_string(), + detail: format!("canonical effect payload decoding failed: {error}"), + denied: false, + }) +} diff --git a/src/ops/env.rs b/src/ops/env.rs index 4a7f1cc..9932896 100644 --- a/src/ops/env.rs +++ b/src/ops/env.rs @@ -1,39 +1,15 @@ //! Thread-local operation environment. -use std::cell::RefCell; -use std::path::PathBuf; +use std::cell::{Cell, RefCell}; use std::rc::Rc; use serde::{Deserialize, Serialize}; -use crate::claims::workspace::WorkspaceCache; -use crate::config::WorkspaceConfig; use crate::error::TexoError; -use crate::host::HostInterface; -use crate::journal_store::JournalStore; -use crate::topology::ResolvedJournal; - -/// Per-invocation environment installed around syncbat handler execution. -pub struct OpEnv { - /// Open workspace store. - pub store: JournalStore, - /// Workspace identifier. - pub workspace_id: String, - /// Workspace root. - pub root: PathBuf, - /// Resolved workspace config. - pub config: WorkspaceConfig, - /// Per-host projection cache. - pub cache: RefCell, - /// Append receipts observed by the effect backend. - pub receipts: RefCell>, - /// Deterministic operation timestamp supplied by surfaces. - pub observed_at_ms: u64, - /// Actual mounted `hostbat` interface for the fingerprint operation. - pub host_interface: HostInterface, - /// Selected physical journal and its authority role. - pub journal: ResolvedJournal, -} + +mod context; + +pub use context::{EnvGuard, OpEnv}; /// Compact receipt note returned by operation JSON outputs. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -48,11 +24,36 @@ pub struct ReceiptNote { thread_local! { static ENV: RefCell>> = const { RefCell::new(None) }; + static REPLAY_DEPTH: Cell = const { Cell::new(0) }; +} + +struct ReplayGuard; + +impl Drop for ReplayGuard { + fn drop(&mut self) { + REPLAY_DEPTH.with(|depth| depth.set(depth.get().saturating_sub(1))); + } } -/// Guard that restores the previous operation environment on drop. -pub struct EnvGuard { - previous: Option>, +/// Execute the common deterministic projection boundary while model transport +/// is denied. All workspace assembly and direct `BatPak` projection entrypoints +/// must pass through this function. +pub(crate) fn deterministic_projection(f: impl FnOnce() -> T) -> T { + REPLAY_DEPTH.with(|depth| depth.set(depth.get().saturating_add(1))); + let guard = ReplayGuard; + let output = f(); + drop(guard); + output +} + +/// Compatibility name for deterministic projection replay. +pub(crate) fn replay_scope(f: impl FnOnce() -> T) -> T { + deterministic_projection(f) +} + +/// True only when the current thread is outside deterministic replay. +pub(crate) fn model_calls_allowed() -> bool { + REPLAY_DEPTH.with(|depth| depth.get() == 0) } /// Install an operation environment for the current thread. @@ -96,7 +97,11 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::topology::{JournalId, JournalRole}; + use crate::claims::workspace::WorkspaceCache; + use crate::config::WorkspaceConfig; + use crate::host::HostInterface; + use crate::journal_store::JournalStore; + use crate::topology::{JournalId, JournalRole, ResolvedJournal}; fn test_env(root: &TempDir, workspace_id: &str) -> Rc { let store = @@ -170,4 +175,15 @@ mod tests { let error = with(|_| ()).expect_err("environment removed"); assert_eq!(error.code(), "op.runtime"); } + + #[test] + fn replay_guard_is_nested_and_restored() { + assert!(model_calls_allowed()); + deterministic_projection(|| { + assert!(!model_calls_allowed()); + deterministic_projection(|| assert!(!model_calls_allowed())); + assert!(!model_calls_allowed()); + }); + assert!(model_calls_allowed()); + } } diff --git a/src/ops/env/context.rs b/src/ops/env/context.rs new file mode 100644 index 0000000..141133c --- /dev/null +++ b/src/ops/env/context.rs @@ -0,0 +1,38 @@ +use std::cell::RefCell; +use std::path::PathBuf; +use std::rc::Rc; + +use crate::claims::workspace::WorkspaceCache; +use crate::config::WorkspaceConfig; +use crate::host::HostInterface; +use crate::journal_store::JournalStore; +use crate::topology::ResolvedJournal; + +use super::ReceiptNote; + +/// Per-invocation environment installed around syncbat handler execution. +pub struct OpEnv { + /// Open workspace store. + pub store: JournalStore, + /// Workspace identifier. + pub workspace_id: String, + /// Workspace root. + pub root: PathBuf, + /// Resolved workspace config. + pub config: WorkspaceConfig, + /// Per-host projection cache. + pub cache: RefCell, + /// Append receipts observed by the effect backend. + pub receipts: RefCell>, + /// Deterministic operation timestamp supplied by surfaces. + pub observed_at_ms: u64, + /// Actual mounted `hostbat` interface for the fingerprint operation. + pub host_interface: HostInterface, + /// Selected physical journal and its authority role. + pub journal: ResolvedJournal, +} + +/// Guard that restores the previous operation environment on drop. +pub struct EnvGuard { + pub(super) previous: Option>, +} diff --git a/src/ops/handlers.rs b/src/ops/handlers.rs index 72daf18..fd66d57 100644 --- a/src/ops/handlers.rs +++ b/src/ops/handlers.rs @@ -1,5132 +1,146 @@ -//! First texo operation handlers. -#![expect( - missing_docs, - reason = "syncbat::operation generates public registration shims without doc injection hooks" -)] - -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt::Write as _; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use batpak::coordinate::Region; -use batpak::event::{EventKind, EventPayload, EventSourced}; -use batpak::id::EntityIdType; -use batpak::store::Freshness; -use serde::{Deserialize, Serialize}; -use syncbat::{CoreBuilder, HandlerError, HandlerResult, OperationRegisterItem}; - -use crate::claims::card::ClaimCard; -use crate::claims::conflict::ConflictCard; -use crate::claims::evidence::{assemble_through as assemble_evidence_through, EvidenceProjection}; -use crate::claims::temporal::{assemble_through as assemble_temporal_through, TemporalProjection}; -use crate::claims::timeline::{ClaimTimeline, TimelineEntry}; -use crate::claims::workspace::{assemble, assemble_through, ClaimView, WorkspaceView}; -use crate::code_index::{ - build as build_code_index, load as load_code_index, persist as persist_code_index, read_scip, - CodeIndexLimits, -}; -use crate::config::{TexoRootConfig, WorkspaceEntry}; -use crate::error::{SnapshotFailureKind, TexoError}; -use crate::events::coordinate::{ - entity_for_claim, entity_for_conflict, entity_for_workspace_meta, scope_for_workspace, -}; -use crate::events::ids::{claim_id_from_parts, ClaimId, SourceId, WorkspaceId}; -use crate::events::machines::{ - transition_record, TransitionCauseV1, CLAIM_EDGES, CLAIM_MACHINE, CONFLICT_EDGES, - CONFLICT_MACHINE, -}; -use crate::events::payloads::{ - ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, CodeIndexRecordedV1, - ConflictOpenedV2, ConflictResolvedV2, EvidenceOccurrenceRecordedV1, - EvidenceReconciliationAcceptedV1, OnboardingCompiledV2, RelationDeferredV1, RelationJudgedV1, - SessionTurnV1, SourceObservedV2, SourceSnapshotRecordedV1, SourceSnapshotRelationV1, - WorkspaceInitializedV2, -}; -use crate::extract::hints::hints_from_line_normalized; -use crate::extract::markdown::{collect_markdown_files, MarkdownDocument}; -use crate::extract::normalize::normalize_line; -use crate::git_source::{ - capture as capture_git, compare_commits, CaptureLimits, CapturedLayer, CapturedSource, -}; -use crate::knowledge::{ - AnalysisQuality, AnswerState, ByteRange, ClaimEvidence, CodeIndexArtifact, CodeIndexId, - CodeOccurrence, CoverageGap, CoverageGapKind, EvidenceLinkMethod, EvidenceOccurrence, - EvidenceOccurrenceId, EvidenceSourceKind, EvidenceStance, KnowledgeCoverage, LineRange, - RepositoryId, SnapshotDescriptor, SnapshotRead, SnapshotToken, TemporalRelation, - TriangulationTarget, UncertaintyReason, MAX_EVIDENCE_EXCERPT_BYTES, +//! Texo operation handlers, organized by operation concern. + +use syncbat::{CoreBuilder, OperationRegisterItem}; + +mod agent_context; +mod claims; +mod common; +mod compile; +mod conflicts; +mod host; +mod ingest; +mod knowledge; +mod knowledge_read; +mod model; +mod relate; +mod render; +mod stats; +mod verify; +mod workspace; + +pub(crate) use common::{ + append_json, assemble_current_view, op_runtime, parse_input, run_op, take_receipts, + workspace_temporal_policy, }; -use crate::ops::env::{self, ReceiptNote}; -use crate::ops::reconcile::append_proposals as append_reconciliation_proposals; -use crate::reconcile::{ - claims_from_view as reconcile_claims, evaluate_with_backends, plan_candidates, - unresolved_row as reconcile_unresolved_row, KnowledgeReconcileInput, KnowledgeReconcileOutput, - ReconcileBackendOutput, ReconcileCompletion, +pub(crate) use ingest::{ + infer_supersessions, plan_sources, ExplicitSupersessionOutcome, SourcePlan, }; -use crate::relate::heuristic; -use crate::semantics::pipeline::{ - receipt_view, ClaimStatus as SemanticClaimStatus, ClaimView as SemanticClaimView, - ParallelRelateOptions, RelateTemporalPolicy, RelateThresholds, -}; - -const WORKSPACE_VIEW_PROJECTION: &str = "texo.workspace.view.v2"; -const CLAIM_EXPLAIN_PROJECTION: &str = "texo.claim.explain.v2"; -const RELATE_PREFILTER: f32 = 0.60; -const MAX_INLINE_SOURCE_FAILURES: usize = 256; -const MAX_SOURCE_FAILURE_DETAIL_CHARS: usize = 512; -const MAX_TRIANGULATION_CODE_OCCURRENCES: usize = 200; -const MAX_TEMPORAL_SNAPSHOT_COMPARISONS: usize = 1_024; -const MAX_GIT_ANCESTRY_WALK: usize = 100_000; -#[cfg(feature = "openrouter")] -const ENV_RELATE_CACHE: &str = "TEXO_RELATE_CACHE"; -#[cfg(feature = "openrouter")] -const DEFAULT_RELATE_CACHE: &str = ".texo/relate-cache"; - -#[syncbat::operation( - descriptor = WORKSPACE_INIT, - register = register_workspace_init, - register_item = workspace_init_item, - name = "texo.workspace.init", - effect = Persist, - input_schema = "texo.workspace.init.input.v2", - output_schema = "texo.workspace.init.output.v2", - receipt_kind = "receipt.texo.workspace.init.v2", - appends_events = ["evt.e007"] -)] -#[tracing::instrument(skip_all)] -fn workspace_init(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.workspace.init", || { - let input: WorkspaceInitInput = parse_input("texo.workspace.init", input)?; - let (root, observed_at_ms) = - env::with(|op_env| (op_env.root.clone(), op_env.observed_at_ms))?; - let config_path = root.join(".texo").join("config.toml"); - - let mut root_config = if config_path.exists() { - TexoRootConfig::load(&config_path).map_err(config_error)? - } else { - TexoRootConfig { - default_workspace: input.workspace_id.clone(), - workspaces: BTreeMap::new(), - gateway: None, - } - }; - root_config - .default_workspace - .clone_from(&input.workspace_id); - if !root_config.workspaces.contains_key(&input.workspace_id) { - root_config.upsert_workspace( - &input.workspace_id, - WorkspaceEntry::for_id(&input.workspace_id), - ); - } - - let raw = toml::to_string_pretty(&root_config).map_err(|error| TexoError::Config { - detail: error.to_string(), - source: Some(Box::new(error)), - })?; - let config_unchanged = std::fs::read(&config_path) - .ok() - .is_some_and(|existing| existing == raw.as_bytes()); - if !config_unchanged { - if let Some(parent) = config_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&config_path, raw.as_bytes())?; - } - let config_digest_hex = blake3::hash(raw.as_bytes()).to_hex().to_string(); - let journal_digest_matches = env::with(|op_env| { - let entity = entity_for_workspace_meta(&input.workspace_id); - let mut entries = op_env.store.by_entity(&entity); - entries.sort_by_key(batpak::store::IndexEntry::global_sequence); - let Some(entry) = entries.last() else { - return Ok::<_, TexoError>(false); - }; - let raw = op_env.store.read_raw(entry.event_id())?; - let payload: WorkspaceInitializedV2 = batpak::encoding::from_bytes(&raw.event.payload) - .map_err(|error| TexoError::Decode { - entity, - detail: error.to_string(), - })?; - Ok(payload.config_digest_hex == config_digest_hex) - })??; - let already_initialized = config_unchanged && journal_digest_matches; - - append_json( - "texo.workspace.init", - cx, - ::KIND, - &WorkspaceInitializedV2 { - workspace_id: input.workspace_id.clone(), - schema: "texo.v2".to_string(), - config_digest_hex, - created_at_ms: observed_at_ms, - }, - )?; - let mut receipts = take_receipts()?; - let receipt = receipts.pop().ok_or_else(|| TexoError::OpRuntime { - op: "texo.workspace.init".to_string(), - detail: "workspace init append produced no receipt".to_string(), - denied: false, - })?; - - Ok(WorkspaceInitOutput { - workspace_id: input.workspace_id, - config_path: config_path.to_string_lossy().to_string(), - already_initialized, - receipt, - }) - }) -} - -#[syncbat::operation( - descriptor = INGEST_RUN, - register = register_ingest_run, - register_item = ingest_run_item, - name = "texo.ingest.run", - effect = Persist, - input_schema = "texo.ingest.run.input.v2", - output_schema = "texo.ingest.run.output.v3", - receipt_kind = "receipt.texo.ingest.run.v2", - appends_events = ["evt.e001", "evt.e002", "evt.e003"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -#[expect( - clippy::too_many_lines, - reason = "ingest planning and append phases stay visibly separated to prove strict atomicity" -)] -fn ingest_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.ingest.run", || { - let input: IngestRunInput = parse_input("texo.ingest.run", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.ingest.run", error))?; - let (root, workspace_id, config) = env::with(|op_env| { - ( - op_env.root.clone(), - op_env.workspace_id.clone(), - op_env.config.clone(), - ) - })?; - let project_started = Instant::now(); - let mut view = assemble_current_view()?; - let mut project_ms = elapsed_ms(project_started); - let path = resolve_path(&root, &input.path); - let plan = plan_sources( - "texo.ingest.run", - &root, - &path, - &workspace_id, - input.observed_at_ms, - config.extractor_cmd.as_deref(), - &view, - )?; - if !plan.skipped.is_empty() && (input.strict || plan.succeeded == 0 && !plan.empty) { - let sample = - serde_json::to_string(&plan.skipped.iter().take(8).cloned().collect::>())?; - let (_, artifact) = - settle_source_failures(&root, input.observed_at_ms, plan.skipped.clone())?; - return Err(TexoError::Source { - path: path.to_string_lossy().to_string(), - detail: format!( - "{} source(s) failed during planning; strict={} good_sources={}; sample={sample}; artifact={}", - plan.skipped.len(), - input.strict, - plan.succeeded, - artifact.as_deref().unwrap_or("inline") - ), - }); - } - - let mut source_count = 0_u32; - let mut claim_count = 0_u32; - let mut supersede_count = 0_u32; - let mut held_supersessions = Vec::new(); - let mut append_ms = 0_u64; - - if input.dry_run { - for source in &plan.sources { - source_count = source_count.saturating_add(1); - let planned = u32::try_from(source.claims.len()).unwrap_or(u32::MAX); - claim_count = claim_count.saturating_add(planned); - } - } else { - let append_started = Instant::now(); - for source in &plan.sources { - append_json( - "texo.ingest.run", - cx, - ::KIND, - &source.observed, - )?; - source_count = source_count.saturating_add(1); - for claim in &source.claims { - append_json( - "texo.ingest.run", - cx, - ::KIND, - claim, - )?; - claim_count = claim_count.saturating_add(1); - } - } - append_ms = append_ms.saturating_add(elapsed_ms(append_started)); - - let project_started = Instant::now(); - view = assemble_current_view()?; - project_ms = project_ms.saturating_add(elapsed_ms(project_started)); - if !config - .semantics - .as_ref() - .is_some_and(|semantics| semantics.enabled) - { - let new_claims = plan - .sources - .iter() - .flat_map(|source| source.claims.iter().cloned()) - .collect::>(); - let temporal = workspace_temporal_policy(&view)?; - let inference = - infer_supersessions(&view, &new_claims, input.observed_at_ms, &temporal)?; - let append_started = Instant::now(); - for superseded in inference.applied { - append_json( - "texo.ingest.run", - cx, - ::KIND, - &superseded, - )?; - supersede_count = supersede_count.saturating_add(1); - } - held_supersessions = inference.held; - append_ms = append_ms.saturating_add(elapsed_ms(append_started)); - } - } - - let outcome = if plan.skipped.is_empty() { - IngestCompletion::Complete - } else { - IngestCompletion::Partial - }; - let skipped_total = plan.skipped.len(); - let (skipped, skipped_artifact) = - settle_source_failures(&root, input.observed_at_ms, plan.skipped)?; - Ok(IngestRunOutput { - outcome, - workspace_id, - sources_observed: source_count, - claims_recorded: claim_count, - claims_superseded: supersede_count, - supersessions_held: held_supersessions.len(), - held_supersessions, - dry_run: input.dry_run, - empty: plan.empty, - skipped, - skipped_total, - skipped_artifact, - phase_ms: IngestPhaseMs { - discover: plan.discover_ms, - extract: plan.extract_ms, - append: append_ms, - project: project_ms, - }, - events_appended: if input.dry_run { - 0 - } else { - u64::from(source_count) - .saturating_add(u64::from(claim_count)) - .saturating_add(u64::from(supersede_count)) - }, - receipts: if input.dry_run { - Vec::new() - } else { - take_receipts()? - }, - }) - }) -} - -#[syncbat::operation( - descriptor = CLAIMS_LIST, - register = register_claims_list, - register_item = claims_list_item, - name = "texo.claims.list", - effect = Inspect, - input_schema = "texo.claims.list.input.v3", - output_schema = "texo.claims.list.output.v3", - receipt_kind = "receipt.texo.claims.list.v3", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn claims_list(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.claims.list", || { - let input: ClaimsListInput = parse_input("texo.claims.list", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.claims.list", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - let claims = claim_list_rows(&view, input.subject.as_deref())?; - Ok(ClaimsListOutput { - workspace_id: view.workspace_id.clone(), - frontier: view.frontier, - claims, - snapshot, - }) - }) -} - -#[syncbat::operation( - descriptor = CLAIMS_SEARCH, - register = register_claims_search, - register_item = claims_search_item, - name = "texo.claims.search", - effect = Inspect, - input_schema = "texo.claims.search.input.v2", - output_schema = "texo.claims.search.output.v2", - receipt_kind = "receipt.texo.claims.search.v2", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn claims_search(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.claims.search", || { - let input: ClaimsSearchInput = parse_input("texo.claims.search", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.claims.search", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - let query = input.query.unwrap_or_default(); - if query.len() > 256 { - return Err(TexoError::OpInput { - op: "texo.claims.search".to_string(), - detail: "query exceeds 256 bytes".to_string(), - }); - } - let limit = input.limit.unwrap_or(25); - if !(1..=100).contains(&limit) { - return Err(TexoError::OpInput { - op: "texo.claims.search".to_string(), - detail: "limit must be between 1 and 100".to_string(), - }); - } - let offset = parse_claim_search_cursor(input.cursor.as_deref())?; - let query_terms = query - .split_whitespace() - .map(str::to_ascii_lowercase) - .collect::>(); - let rows = claim_list_rows(&view, input.subject.as_deref())? - .into_iter() - .filter(|row| input.status.is_none_or(|status| row.status == status)) - .filter(|row| claim_matches_query(row, &query_terms)) - .collect::>(); - let total = rows.len(); - let page = rows - .into_iter() - .skip(offset) - .take(limit) - .collect::>(); - let returned = page.len(); - let next_offset = offset.saturating_add(returned); - let has_more = next_offset < total; - Ok(ClaimsSearchOutput { - workspace_id: view.workspace_id.clone(), - frontier: view.frontier, - freshness: view.freshness, - total, - returned, - has_more, - next_cursor: has_more.then(|| format!("texo-claims-v1:{next_offset}")), - claims: page, - snapshot, - }) - }) -} - -#[syncbat::operation( - descriptor = KNOWLEDGE_SEARCH, - register = register_knowledge_search, - register_item = knowledge_search_item, - name = "texo.knowledge.search", - effect = Inspect, - input_schema = "texo.knowledge.search.input.v1", - output_schema = "texo.knowledge.search.output.v1", - receipt_kind = "receipt.texo.knowledge.search.v1", - reads_events = ["evt.e00e"], - queries_projections = ["texo.workspace.view.v2", "texo.code.index.v1"] -)] -#[tracing::instrument(skip_all)] -fn knowledge_search(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.knowledge.search", || { - let input: KnowledgeSearchInput = parse_input("texo.knowledge.search", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.knowledge.search", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - search_knowledge_from_view(&view, snapshot, &input) - }) -} - -#[syncbat::operation( - descriptor = CLAIM_EXPLAIN, - register = register_claim_explain, - register_item = claim_explain_item, - name = "texo.claim.explain", - effect = Inspect, - input_schema = "texo.claim.explain.input.v3", - output_schema = "texo.claim.explain.output.v4", - receipt_kind = "receipt.texo.claim.explain.v4", - queries_projections = ["texo.claim.explain.v2"] -)] -#[tracing::instrument(skip_all)] -fn claim_explain(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.claim.explain", || { - let input: ClaimExplainInput = parse_input("texo.claim.explain", input)?; - cx.projection_read_handle() - .query_projection(CLAIM_EXPLAIN_PROJECTION) - .map_err(|error| op_runtime("texo.claim.explain", error))?; - let entity = entity_for_claim(&input.claim_id); - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - let card = view - .claims - .iter() - .find(|claim| claim.card.claim_id == input.claim_id) - .map(|claim| claim.card.as_ref().clone()) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity.clone(), - })?; - let timeline = claim_timeline_through(&entity, view.frontier)?; - let evidence = evidence_projection_through(view.frontier)?.take_claim(&input.claim_id); - let coverage = coverage_for_view(&view, &snapshot)?; - let answer_state = answer_state_for_claim( - view.claims - .iter() - .find(|claim| claim.card.claim_id == input.claim_id) - .map(|claim| claim.status), - &evidence, - ); - Ok(ClaimExplainOutput { - card, - timeline: timeline.entries, - answer_state, - evidence, - coverage, - snapshot, - }) - }) -} - -#[syncbat::operation( - descriptor = KNOWLEDGE_TRIANGULATE, - register = register_knowledge_triangulate, - register_item = knowledge_triangulate_item, - name = "texo.knowledge.triangulate", - effect = Inspect, - input_schema = "texo.knowledge.triangulate.input.v1", - output_schema = "texo.knowledge.triangulate.output.v1", - receipt_kind = "receipt.texo.knowledge.triangulate.v1", - reads_events = ["evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e"], - queries_projections = ["texo.workspace.view.v2", "texo.evidence.view.v1"] -)] -#[tracing::instrument(skip_all)] -fn knowledge_triangulate(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.knowledge.triangulate", || { - let input: KnowledgeTriangulateInput = parse_input("texo.knowledge.triangulate", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.knowledge.triangulate", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - triangulate_from_view(&view, &snapshot, input.target) - }) -} - -#[syncbat::operation( - descriptor = CLAIM_SUPERSEDE, - register = register_claim_supersede, - register_item = claim_supersede_item, - name = "texo.claim.supersede", - effect = Persist, - input_schema = "texo.claim.supersede.input.v2", - output_schema = "texo.claim.supersede.output.v2", - receipt_kind = "receipt.texo.claim.supersede.v2", - appends_events = ["evt.e003"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn claim_supersede(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.claim.supersede", || { - let input: ClaimSupersedeInput = parse_input("texo.claim.supersede", input)?; - if input.old == input.new { - return Err(TexoError::OpInput { - op: "texo.claim.supersede".to_string(), - detail: "old and new claims must differ".to_string(), - }); - } - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.claim.supersede", error))?; - - let old_entity = entity_for_claim(&input.old); - let new_entity = entity_for_claim(&input.new); - let (old_card, new_card, workspace_id) = env::with(|op_env| { - let old_card = op_env - .store - .project::(&old_entity, &Freshness::Consistent)?; - let new_card = op_env - .store - .project::(&new_entity, &Freshness::Consistent)?; - Ok::<_, TexoError>((old_card, new_card, op_env.workspace_id.clone())) - })??; - let old_card = old_card.ok_or_else(|| TexoError::MissingEntity { - entity: old_entity.clone(), - })?; - let _new_card = new_card.ok_or_else(|| TexoError::MissingEntity { - entity: new_entity.clone(), - })?; - if old_card.phase == 2 && old_card.superseded_by.as_deref() == Some(input.new.as_str()) { - return Ok(ClaimSupersedeOutput { - old: input.old, - new: input.new, - already_applied: true, - receipt: None, - }); - } - if old_card.phase != 1 { - return Err(TexoError::Transition { - machine: CLAIM_MACHINE.to_string(), - from: old_card.phase, - to: 2, - context: Some(format!( - "claim {} is already {}{}", - input.old, - claim_phase_name(old_card.phase), - old_card - .superseded_by - .as_deref() - .map_or_else(String::new, |successor| format!(" by {successor}")) - )), - }); - } - - let payload = ClaimSupersededV2 { - old_claim_id: input.old.clone(), - new_claim_id: input.new.clone(), - workspace_id, - reason: input.reason, - decided_by: input.decided_by, - observed_at_ms: input.observed_at_ms, - transition: transition_record( - CLAIM_MACHINE, - &old_entity, - 1, - 2, - vec![TransitionCauseV1 { - lane: 0, - key: format!("claim:{}", input.new), - }], - input.observed_at_ms, - ), - }; - append_json( - "texo.claim.supersede", - cx, - ::KIND, - &payload, - )?; - Ok(ClaimSupersedeOutput { - old: input.old, - new: input.new, - already_applied: false, - receipt: Some(take_one_receipt("texo.claim.supersede")?), - }) - }) -} - -#[syncbat::operation( - descriptor = VERIFY_RUN, - register = register_verify_run, - register_item = verify_run_item, - name = "texo.verify.run", - effect = Inspect, - input_schema = "texo.verify.run.input.v2", - output_schema = "texo.verify.run.output.v2", - receipt_kind = "receipt.texo.verify.run.v2", - reads_events = ["evt.e001", "evt.e002", "evt.e003", "evt.e004", "evt.e005", "evt.e006", "evt.e007", "evt.e008", "evt.e009", "evt.e00a", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e", "evt.e00f", "evt.e010"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn verify_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.verify.run", || { - let _input: VerifyRunInput = parse_input("texo.verify.run", input)?; - let replay_started = Instant::now(); - for kind in DOMAIN_KINDS { - cx.event_read_handle() - .read_event(format!("evt.{:04x}", kind.as_raw_u16())) - .map_err(|error| op_runtime("texo.verify.run", error))?; - } - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.verify.run", error))?; - - let mut errors = Vec::new(); - let (journal_ok, view, events_replayed) = env::with(|op_env| { - let chain = op_env.store.verify_chain()?; - let mut journal_ok = chain.is_intact(); - if !chain.is_intact() { - errors.push(format!("chain: {chain:?}")); - } - let scope = scope_for_workspace(&op_env.workspace_id); - let region = Region::scope(&scope); - let mut after = None; - let mut events_replayed = 0usize; - loop { - let page = op_env.store.query_entries_after(®ion, after, 256); - if page.is_empty() { - break; - } - for entry in &page { - events_replayed = events_replayed.saturating_add(1); - if !DOMAIN_KINDS.contains(&entry.event_kind()) { - journal_ok = false; - errors.push(format!( - "unknown event kind evt.{:04x} at {}", - entry.event_kind().as_raw_u16(), - entry.global_sequence() - )); - } - if let Err(error) = op_env.store.read_raw(entry.event_id()) { - journal_ok = false; - errors.push(format!( - "decode {}: {error}", - event_id_hex(entry.event_id()) - )); - } - } - after = page.last().map(batpak::store::IndexEntry::global_sequence); - } - let mut cache = op_env.cache.borrow_mut(); - let view = assemble(&op_env.store, &op_env.workspace_id, &mut cache)?; - Ok::<_, TexoError>((journal_ok, view, events_replayed)) - })??; - - let projection_ok = view - .claims - .iter() - .all(|claim| claim.card.anomalies.is_empty()) - && view - .conflicts - .iter() - .all(|conflict| conflict.anomalies.is_empty()); - if !projection_ok { - errors.push("projection anomalies present".to_string()); - } - let transitions_ok = validate_transition_edges(&view, &mut errors); - - Ok(VerifyRunOutput { - projection_ok, - journal_ok, - transitions_ok, - errors, - replay_ms: elapsed_ms(replay_started), - events_replayed, - }) - }) -} - -#[syncbat::operation( - descriptor = STALENESS_CHECK, - register = register_staleness_check, - register_item = staleness_check_item, - name = "texo.staleness.check", - effect = Inspect, - input_schema = "texo.staleness.check.input.v3", - output_schema = "texo.staleness.check.output.v3", - receipt_kind = "receipt.texo.staleness.check.v3", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn staleness_check(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.staleness.check", || { - let input: StalenessCheckInput = parse_input("texo.staleness.check", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.staleness.check", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - let (root, workspace_id) = - env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; - let path = resolve_path(&root, &input.path); - check_staleness_from_view(&view, &workspace_id, &root, &path, snapshot) - }) -} - -#[syncbat::operation( - descriptor = CONTEXT_AGENT, - register = register_context_agent, - register_item = context_agent_item, - name = "texo.context.agent", - effect = Inspect, - input_schema = "texo.context.agent.input.v3", - output_schema = "texo.context.agent.output.v3", - receipt_kind = "receipt.texo.context.agent.v3", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn context_agent(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.context.agent", || { - let input: ContextAgentInput = parse_input("texo.context.agent", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.context.agent", error))?; - if input.strict_settlement { - require_complete_settlement()?; - } - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - build_agent_context_from_view( - &view, - input.subject.as_deref(), - input.include_stale, - snapshot, - ) - }) -} - -#[syncbat::operation( - descriptor = COMPILE_RUN, - register = register_compile_run, - register_item = compile_run_item, - name = "texo.compile.run", - effect = Persist, - input_schema = "texo.compile.run.input.v2", - output_schema = "texo.compile.run.output.v2", - receipt_kind = "receipt.texo.compile.run.v2", - appends_events = ["evt.e005"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn compile_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.compile.run", || { - let input: CompileRunInput = parse_input("texo.compile.run", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.compile.run", error))?; - if input.strict_settlement { - require_complete_settlement()?; - } - let view = assemble_current_view()?; - let snapshot = snapshot_for_view(&view)?; - let context = build_agent_context_from_view(&view, None, true, snapshot.clone())?; - let conflict_report = heuristic::detect_conflicts(&view)?; - let (root, workspace_id) = - env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; - let out_dir = resolve_path(&root, &input.out_dir); - let stale_report = StalenessReport { - workspace_id: workspace_id.clone(), - checked_path: ".".to_string(), - replayed_through_sequence: view.frontier, - diagnostics: Vec::new(), - snapshot, - }; - let files = compile_artifacts(&context, &view, &stale_report, &conflict_report)?; - for file in &files { - let path = out_dir.join(&file.name); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&path, file.contents.as_bytes())?; - } - let source_claim_ids = view - .claims - .iter() - .map(|claim| claim.card.claim_id.clone()) - .collect::>(); - let doc_id = format!( - "doc_{}", - &blake3::hash(out_dir.to_string_lossy().as_bytes()).to_hex()[..12] - ); - append_json( - "texo.compile.run", - cx, - ::KIND, - &OnboardingCompiledV2 { - doc_id, - workspace_id, - output_path: input.out_dir.to_string_lossy().to_string(), - source_claim_ids, - replayed_through_sequence: view.frontier, - compiled_at_ms: input.observed_at_ms, - }, - )?; - Ok(CompileRunOutput { - files: files.into_iter().map(|file| file.name).collect::>(), - receipt: take_one_receipt("texo.compile.run")?, - }) - }) -} - -#[syncbat::operation( - descriptor = CONFLICTS_LIST, - register = register_conflicts_list, - register_item = conflicts_list_item, - name = "texo.conflicts.list", - effect = Inspect, - input_schema = "texo.conflicts.list.input.v2", - output_schema = "texo.conflicts.list.output.v2", - receipt_kind = "receipt.texo.conflicts.list.v2", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn conflicts_list(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.conflicts.list", || { - let _input: ConflictsListInput = parse_input("texo.conflicts.list", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.conflicts.list", error))?; - let view = assemble_current_view()?; - Ok(conflicts_output(&view)) - }) -} - -#[syncbat::operation( - descriptor = CONFLICTS_COMMIT, - register = register_conflicts_commit, - register_item = conflicts_commit_item, - name = "texo.conflicts.commit", - effect = Persist, - input_schema = "texo.conflicts.commit.input.v2", - output_schema = "texo.conflicts.commit.output.v2", - receipt_kind = "receipt.texo.conflicts.commit.v2", - appends_events = ["evt.e004"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn conflicts_commit(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.conflicts.commit", || { - let input: ConflictsCommitInput = parse_input("texo.conflicts.commit", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.conflicts.commit", error))?; - let view = assemble_current_view()?; - let detected = heuristic::detect_conflicts(&view)?; - let existing = view - .conflicts - .iter() - .map(|conflict| conflict.conflict_id.clone()) - .collect::>(); - let mut committed = Vec::new(); - for entry in detected.conflicts { - if existing.contains(&entry.conflict_id) { - continue; - } - let newer = newer_claim(&view, &entry.claim_a, &entry.claim_b)?; - append_json( - "texo.conflicts.commit", - cx, - ::KIND, - &ConflictOpenedV2 { - conflict_id: entry.conflict_id.clone(), - workspace_id: view.workspace_id.clone(), - claim_a: entry.claim_a.clone(), - claim_b: entry.claim_b.clone(), - reason: entry.reason.clone(), - detector: "heuristic-v1".to_string(), - observed_at_ms: input.observed_at_ms, - transition: transition_record( - CONFLICT_MACHINE, - &entity_for_conflict(&entry.conflict_id), - 0, - 1, - vec![TransitionCauseV1 { - lane: 0, - key: format!("ingest:{}", newer.source_id), - }], - input.observed_at_ms, - ), - }, - )?; - let receipt = take_one_receipt("texo.conflicts.commit")?; - committed.push(CommittedConflict { - conflict_id: entry.conflict_id, - sequence: receipt.global_sequence, - receipt, - }); - } - Ok(committed) - }) -} - -#[syncbat::operation( - descriptor = CONFLICT_RESOLVE, - register = register_conflict_resolve, - register_item = conflict_resolve_item, - name = "texo.conflict.resolve", - effect = Persist, - input_schema = "texo.conflict.resolve.input.v2", - output_schema = "texo.conflict.resolve.output.v2", - receipt_kind = "receipt.texo.conflict.resolve.v2", - appends_events = ["evt.e006"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn conflict_resolve(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.conflict.resolve", || { - let input: ConflictResolveInput = parse_input("texo.conflict.resolve", input)?; - if input.resolution != "resolved" && input.resolution != "ignored" { - return Err(TexoError::StatusParse { - value: input.resolution, - }); - } - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.conflict.resolve", error))?; - let entity = entity_for_conflict(&input.conflict_id); - let card = env::with(|op_env| { - op_env - .store - .project::(&entity, &Freshness::Consistent) - })??; - let card = card.ok_or_else(|| TexoError::MissingEntity { - entity: entity.clone(), - })?; - let target_phase = if input.resolution == "resolved" { 2 } else { 3 }; - if card.phase == target_phase { - return Ok(ConflictResolveOutput { - conflict_id: input.conflict_id, - resolution: input.resolution, - already_applied: true, - receipt: None, - }); - } - if card.phase != 1 { - return Err(TexoError::Transition { - machine: CONFLICT_MACHINE.to_string(), - from: card.phase, - to: target_phase, - context: Some(format!( - "conflict {} is already {}", - input.conflict_id, - conflict_phase_name(card.phase) - )), - }); - } - append_json( - "texo.conflict.resolve", - cx, - ::KIND, - &ConflictResolvedV2 { - conflict_id: input.conflict_id.clone(), - workspace_id: card.workspace_id, - resolution: input.resolution.clone(), - resolved_by: input.resolved_by, - observed_at_ms: input.observed_at_ms, - transition: transition_record( - CONFLICT_MACHINE, - &entity, - 1, - target_phase, - vec![TransitionCauseV1 { - lane: 0, - key: format!("conflict:{}", input.conflict_id), - }], - input.observed_at_ms, - ), - }, - )?; - Ok(ConflictResolveOutput { - conflict_id: input.conflict_id, - resolution: input.resolution, - already_applied: false, - receipt: Some(take_one_receipt("texo.conflict.resolve")?), - }) - }) -} - -#[syncbat::operation( - descriptor = RELATE_RUN, - register = register_relate_run, - register_item = relate_run_item, - name = "texo.relate.run", - effect = Persist, - input_schema = "texo.relate.run.input.v2", - output_schema = "texo.relate.run.output.v2", - receipt_kind = "receipt.texo.relate.run.v2", - appends_events = ["evt.e003", "evt.e004", "evt.e009", "evt.e00a"], - queries_projections = ["texo.workspace.view.v2"], - requires_capabilities = ["texo.cap.model"] -)] -#[tracing::instrument(skip_all)] -fn relate_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.relate.run", || { - let input: RelateRunInput = parse_input("texo.relate.run", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.relate.run", error))?; - run_relate_pass("texo.relate.run", cx, input.observed_at_ms, input.strict) - }) -} - -#[syncbat::operation( - descriptor = HOST_FINGERPRINT, - register = register_host_fingerprint, - register_item = host_fingerprint_item, - name = "texo.host.fingerprint", - effect = Inspect, - input_schema = "texo.host.fingerprint.input.v2", - output_schema = "texo.host.fingerprint.output.v2", - receipt_kind = "receipt.texo.host.fingerprint.v2" -)] -#[tracing::instrument(skip_all)] -fn host_fingerprint(input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.host.fingerprint", || { - let _input: HostFingerprintInput = parse_input("texo.host.fingerprint", input)?; - env::with(|op_env| op_env.host_interface.clone()) - }) -} - -#[syncbat::operation( - descriptor = KNOWLEDGE_INDEX, - register = register_knowledge_index, - register_item = knowledge_index_item, - name = "texo.knowledge.index", - effect = Persist, - input_schema = "texo.knowledge.index.input.v1", - output_schema = "texo.knowledge.index.output.v2", - receipt_kind = "receipt.texo.knowledge.index.v1", - appends_events = ["evt.e003", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00f"], - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn knowledge_index(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.knowledge.index", || { - let input: KnowledgeIndexInput = parse_input("texo.knowledge.index", input)?; - let limits = input.validated_limits()?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.knowledge.index", error))?; - let view = assemble_current_view()?; - let (root, workspace_id) = - env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; - let workspace = WorkspaceId::new(workspace_id.clone())?; - let previous_snapshots = source_snapshots_through(Some(view.frontier))?; - let repository_id = previous_snapshots.last().cloned().map_or_else( - || { - let canonical = std::fs::canonicalize(&root).unwrap_or_else(|_| root.clone()); - RepositoryId::derive(&format!( - "texo.repository.v1\u{1f}{workspace_id}\u{1f}{}", - canonical.display() - )) - }, - |snapshot| snapshot.repository_id, - ); - let mut capture = capture_git(&root, repository_id, limits)?; - let already_indexed = latest_source_snapshot(Some(view.frontier))? - .is_some_and(|existing| existing.snapshot_id == capture.snapshot_id); - - let planned = plan_claim_evidence( - &view, - &capture.sources, - &capture.snapshot_id, - input.observed_at_ms, - )?; - for gap in planned.gaps { - if capture.coverage.gaps.len() < 256 { - capture.coverage.gaps.push(gap); - } else { - capture.coverage.truncated = true; - } - } - if !planned.rows.is_empty() { - capture.coverage.analysis_quality = AnalysisQuality::Syntactic; - } - capture.coverage.occurrences = u64::try_from(planned.rows.len()).unwrap_or(u64::MAX); - let relations = plan_and_attach_snapshot_relations( - &root, - &workspace, - &mut capture, - &previous_snapshots, - input.observed_at_ms, - )?; - let snapshot = SourceSnapshotRecordedV1 { - workspace_id: workspace.clone(), - repository_id: capture.repository_id, - snapshot_id: capture.snapshot_id.clone(), - base_commit: capture.base_commit.clone(), - base_tree: capture.base_tree, - index_digest_hex: capture.index_digest_hex, - overlay_digest_hex: capture.overlay_digest_hex, - dirty: capture.dirty, - coverage: capture.coverage.clone(), - capture_max_files: u64::try_from(limits.max_files).unwrap_or(u64::MAX), - capture_max_file_bytes: limits.max_file_bytes, - capture_max_total_bytes: limits.max_total_bytes, - observed_at_ms: input.observed_at_ms, - }; - let indexed_claim_ids = planned - .rows - .iter() - .map(|(_, link)| link.claim_id.clone()) - .collect::>(); - let mut receipts = append_knowledge_plan( - cx, - &workspace, - &snapshot, - &planned.rows, - &relations, - input.observed_at_ms, - )?; - let supersessions = settle_indexed_explicit_supersessions( - cx, - &view, - &indexed_claim_ids, - input.observed_at_ms, - &mut receipts, - )?; - Ok(KnowledgeIndexOutput { - workspace_id, - snapshot_id: capture.snapshot_id, - base_commit: capture.base_commit, - dirty: capture.dirty, - sources_captured: capture.sources.len(), - evidence_recorded: planned.rows.len(), - claims_linked: planned.rows.len(), - relations_recorded: relations.len(), - supersessions_applied: supersessions.applied.len(), - supersessions_held: supersessions.held.len(), - held_supersessions: supersessions.held, - already_indexed, - coverage: capture.coverage, - receipts, - }) - }) -} - -#[syncbat::operation( - descriptor = CODE_INDEX_BUILD, - register = register_code_index_build, - register_item = code_index_build_item, - name = "texo.code.index.build", - effect = Persist, - input_schema = "texo.code.index.build.input.v1", - output_schema = "texo.code.index.build.output.v2", - receipt_kind = "receipt.texo.code.index.build.v1", - appends_events = ["evt.e00e"], - reads_events = ["evt.e00b", "evt.e00e"], - queries_projections = ["texo.code.index.v1"] -)] -#[tracing::instrument(skip_all)] -fn code_index_build(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.code.index.build", || { - let input: CodeIndexBuildInput = parse_input("texo.code.index.build", input)?; - let limits = input.validated_limits()?; - let (root, workspace_id) = - env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; - // Select by the requested id, not "the latest recorded": revisiting a - // previously recorded commit re-derives its existing snapshot id (the - // record is idempotent), and requiring it to also be the newest one would - // reject the id `knowledge.index` just returned whenever an intervening - // snapshot exists. Absent an explicit id, fall back to the latest. - let recorded = match input.snapshot_id.as_ref() { - Some(wanted) => source_snapshot_by_id(wanted)?.ok_or_else(|| TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "requested source snapshot is not recorded; run `texo index` first" - .to_string(), - })?, - None => latest_source_snapshot(None)?.ok_or_else(|| TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "run `texo index` to record a Git source snapshot first".to_string(), - })?, - }; - // Reproduce the snapshot with the *recorded* bounds, not the defaults: a - // snapshot captured under non-default limits would otherwise omit different - // sources here and be misreported as "Git changed" on an unchanged tree. - let recorded_limits = recorded_capture_limits(&recorded); - let capture = capture_git(&root, recorded.repository_id, recorded_limits)?; - if capture.snapshot_id != recorded.snapshot_id { - return Err(TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "Git commit/index/worktree changed; run source indexing again before code indexing".to_string(), - }); - } - if input.can_reuse_default() { - if let Some(existing) = latest_code_index(None, &recorded.snapshot_id)? { - if let Some(artifact) = - load_code_index(&root, &existing.index_id, &existing.artifact_digest_hex)? - { - if artifact.snapshot_id != recorded.snapshot_id { - return Err(TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "code-index artifact belongs to a different source snapshot" - .to_string(), - }); - } - return Ok(CodeIndexBuildOutput { - workspace_id, - snapshot_id: existing.snapshot_id, - index_id: existing.index_id.clone(), - format: artifact.format, - analyzer_fingerprint: artifact.analyzer_fingerprint, - artifact_digest_hex: existing.artifact_digest_hex, - artifact_path: format!( - ".texo/cache/code-index/{}.bin", - existing.index_id.as_str() - ), - coverage: artifact.coverage, - already_indexed: true, - receipt: None, - }); - } - } - } - let scip_bytes = input - .scip_path - .as_deref() - .map(|path| read_scip(&root, path, limits.max_scip_bytes)) - .transpose()?; - let prepared = build_code_index(&capture, scip_bytes.as_deref(), limits)?; - let artifact_path = persist_code_index(&root, &prepared)?; - let payload = CodeIndexRecordedV1 { - workspace_id: WorkspaceId::new(workspace_id.clone())?, - snapshot_id: recorded.snapshot_id, - index_id: prepared.artifact.index_id.clone(), - format: prepared.artifact.format, - analyzer_fingerprint: prepared.artifact.analyzer_fingerprint.clone(), - artifact_digest_hex: prepared.artifact_digest_hex.clone(), - coverage: prepared.artifact.coverage.clone(), - observed_at_ms: input.observed_at_ms, - }; - append_json( - "texo.code.index.build", - cx, - ::KIND, - &payload, - )?; - let relative_artifact = artifact_path - .strip_prefix(&root) - .unwrap_or(&artifact_path) - .to_string_lossy() - .to_string(); - Ok(CodeIndexBuildOutput { - workspace_id, - snapshot_id: payload.snapshot_id, - index_id: payload.index_id, - format: payload.format, - analyzer_fingerprint: payload.analyzer_fingerprint, - artifact_digest_hex: payload.artifact_digest_hex, - artifact_path: relative_artifact, - coverage: payload.coverage, - already_indexed: false, - receipt: Some(take_one_receipt("texo.code.index.build")?), - }) - }) -} - -#[syncbat::operation( - descriptor = KNOWLEDGE_RECONCILE, - register = register_knowledge_reconcile, - register_item = knowledge_reconcile_item, - name = "texo.knowledge.reconcile", - effect = Persist, - input_schema = "texo.knowledge.reconcile.input.v1", - output_schema = "texo.knowledge.reconcile.output.v1", - receipt_kind = "receipt.texo.knowledge.reconcile.v1", - appends_events = ["evt.e00c", "evt.e00d", "evt.e010"], - reads_events = ["evt.e002", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e"], - queries_projections = ["texo.workspace.view.v2", "texo.code.index.v1"], - requires_capabilities = ["texo.cap.model"] -)] -#[tracing::instrument(skip_all)] -fn knowledge_reconcile(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.knowledge.reconcile", || { - let input: KnowledgeReconcileInput = parse_input("texo.knowledge.reconcile", input)?; - let (limits, budget_secs, concurrency) = input.validated()?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.knowledge.reconcile", error))?; - let view = assemble_current_view()?; - let snapshot = - latest_source_snapshot(Some(view.frontier))?.ok_or_else(|| TexoError::OpInput { - op: "texo.knowledge.reconcile".to_string(), - detail: "no source snapshot exists; run `texo index` first".to_string(), - })?; - let loaded = load_code_artifact_at(view.frontier, &snapshot.snapshot_id)?; - let artifact = loaded.artifact.ok_or_else(|| TexoError::OpInput { - op: "texo.knowledge.reconcile".to_string(), - detail: "the current source snapshot has no available code index; run `texo index`" - .to_string(), - })?; - let claims = reconcile_claims(&view)?; - let plan = plan_candidates(&claims, &artifact, limits); - let candidates_considered = plan.candidates.len(); - let evidence = evidence_projection_through(view.frontier)?; - let mut already_linked = 0; - let candidates = plan - .candidates - .into_iter() - .filter(|candidate| { - let linked = evidence - .for_claim(candidate.claim_id.as_str()) - .iter() - .any(|item| { - item.occurrence.occurrence_id == candidate.occurrence.occurrence_id - }); - already_linked += usize::from(linked); - !linked - }) - .collect::>(); - let (root, gateway) = - env::with(|op_env| (op_env.root.clone(), op_env.config.gateway.clone()))?; - let backend = evaluate_with_backends( - &root, - gateway.as_ref(), - &candidates, - std::time::Duration::from_secs(budget_secs), - concurrency, - )?; - let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; - let ReconcileBackendOutput { - proposals, - unresolved: backend_unresolved, - judge_fingerprint, - } = backend; - let (accepted, rejected) = append_reconciliation_proposals( - cx, - &workspace_id, - input.observed_at_ms, - limits.min_score_ppm, - &judge_fingerprint, - proposals, - )?; - let unresolved = backend_unresolved - .iter() - .map(reconcile_unresolved_row) - .collect::>(); - let mut coverage = artifact.coverage; - if plan.truncated { - coverage.truncated = true; - if !coverage - .gaps - .iter() - .any(|gap| gap.kind == CoverageGapKind::BudgetExceeded) - { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::BudgetExceeded, - }); - } - } - let partial = coverage.truncated || !coverage.gaps.is_empty() || !unresolved.is_empty(); - Ok(KnowledgeReconcileOutput { - outcome: if partial { - ReconcileCompletion::Partial - } else { - ReconcileCompletion::Complete - }, - snapshot_id: snapshot.snapshot_id, - candidates_considered, - already_linked, - accepted, - rejected, - unresolved, - coverage, - receipts: take_receipts()?, - }) - }) -} - -#[syncbat::operation( - descriptor = STATS_READ, - register = register_stats_read, - register_item = stats_read_item, - name = "texo.stats.read", - effect = Inspect, - input_schema = "texo.stats.read.input.v1", - output_schema = "texo.stats.read.output.v1", - receipt_kind = "receipt.texo.stats.read.v1", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn stats_read(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.stats.read", || { - let _input: StatsReadInput = parse_input("texo.stats.read", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.stats.read", error))?; - let view = assemble_current_view()?; - let (root, config, journal) = env::with(|op_env| { - ( - op_env.root.clone(), - op_env.config.clone(), - op_env.journal.clone(), - ) - })?; - let store_path = config.store_path_buf(&root); - let projection_path = root - .join(".texo/cache/workspace-view") - .join(format!("{}--{}.bin", view.workspace_id, journal.id)); - let context = build_agent_context_from_view(&view, None, true, snapshot_for_view(&view)?)?; - let agent_context_bytes = serde_json::to_vec(&context)?.len(); - Ok(StatsReadOutput { - journal_id: journal.id, - journal_role: journal.role, - claims_total: view.claims.len(), - events_total: workspace_event_count()?, - journal_bytes: journal_file_bytes(&store_path)?, - projection_bytes: file_bytes(&projection_path)?, - agent_context_bytes: u64::try_from(agent_context_bytes).unwrap_or(u64::MAX), - frontier_sequence: view.frontier, - }) - }) -} - -#[syncbat::operation( - descriptor = WORKSPACE_STATUS, - register = register_workspace_status, - register_item = workspace_status_item, - name = "texo.workspace.status", - effect = Inspect, - input_schema = "texo.workspace.status.input.v2", - output_schema = "texo.workspace.status.output.v2", - receipt_kind = "receipt.texo.workspace.status.v2", - queries_projections = ["texo.workspace.view.v2"] -)] -#[tracing::instrument(skip_all)] -fn workspace_status(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { - run_op("texo.workspace.status", || { - let input: WorkspaceStatusInput = parse_input("texo.workspace.status", input)?; - cx.projection_read_handle() - .query_projection(WORKSPACE_VIEW_PROJECTION) - .map_err(|error| op_runtime("texo.workspace.status", error))?; - let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; - let settlement = authoritative_settlements(Some(view.frontier))?; - let unresolved_pairs = settlement.unresolved_pairs; - let (coverage, code_index_available) = status_coverage(&view, &snapshot)?; - let journal = env::with(|op_env| op_env.journal.clone())?; - Ok(WorkspaceStatusOutput { - workspace_id: view.workspace_id.clone(), - journal_id: journal.id, - journal_role: journal.role, - source_journal: journal.source_journal, - replica_mode: journal.replica_mode, - frontier: view.frontier, - freshness: view.freshness, - claims_total: view.claims.len(), - open_conflicts: view.conflicts.iter().filter(|card| card.phase == 1).count(), - settlement_complete: unresolved_pairs == 0, - unresolved_pairs, - authority_warnings: settlement.warnings.len(), - code_index_available, - coverage, - snapshot, - }) - }) -} - -const DOMAIN_KINDS: [EventKind; 16] = [ - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, - ::KIND, +pub use ingest::{ExplicitSupersessionHoldReason, HeldExplicitSupersession}; +pub(crate) use relate::{run_relate_pass, RelatePassOptions}; + +struct OperationBinding { + item: fn() -> OperationRegisterItem, + register: for<'a> fn(&'a mut CoreBuilder) -> Result<&'a mut CoreBuilder, syncbat::BuildError>, +} + +const OPERATIONS: &[OperationBinding] = &[ + OperationBinding { + item: workspace::workspace_init_item, + register: workspace::register_workspace_init, + }, + OperationBinding { + item: ingest::ingest_run_item, + register: ingest::register_ingest_run, + }, + OperationBinding { + item: claims::claims_list_item, + register: claims::register_claims_list, + }, + OperationBinding { + item: claims::claims_search_item, + register: claims::register_claims_search, + }, + OperationBinding { + item: claims::knowledge_search_item, + register: claims::register_knowledge_search, + }, + OperationBinding { + item: claims::claim_explain_item, + register: claims::register_claim_explain, + }, + OperationBinding { + item: knowledge::knowledge_triangulate_item, + register: knowledge::register_knowledge_triangulate, + }, + OperationBinding { + item: claims::claim_supersede_item, + register: claims::register_claim_supersede, + }, + OperationBinding { + item: verify::verify_run_item, + register: verify::register_verify_run, + }, + OperationBinding { + item: render::staleness_check_item, + register: render::register_staleness_check, + }, + OperationBinding { + item: agent_context::context_agent_item, + register: agent_context::register_context_agent, + }, + OperationBinding { + item: compile::compile_run_item, + register: compile::register_compile_run, + }, + OperationBinding { + item: conflicts::conflicts_list_item, + register: conflicts::register_conflicts_list, + }, + OperationBinding { + item: conflicts::conflicts_commit_item, + register: conflicts::register_conflicts_commit, + }, + OperationBinding { + item: conflicts::conflict_resolve_item, + register: conflicts::register_conflict_resolve, + }, + OperationBinding { + item: relate::relate_run_item, + register: relate::register_relate_run, + }, + OperationBinding { + item: host::host_fingerprint_item, + register: host::register_host_fingerprint, + }, + OperationBinding { + item: knowledge::knowledge_index_item, + register: knowledge::register_knowledge_index, + }, + OperationBinding { + item: knowledge::code_index_build_item, + register: knowledge::register_code_index_build, + }, + OperationBinding { + item: knowledge::knowledge_reconcile_item, + register: knowledge::register_knowledge_reconcile, + }, + OperationBinding { + item: stats::stats_read_item, + register: stats::register_stats_read, + }, + OperationBinding { + item: workspace::workspace_status_item, + register: workspace::register_workspace_status, + }, ]; -/// Return the operation registration items. +/// Return the operation catalog in deterministic registration order. #[must_use] pub fn catalog() -> Vec { - vec![ - workspace_init_item(), - ingest_run_item(), - claims_list_item(), - claims_search_item(), - knowledge_search_item(), - claim_explain_item(), - claim_supersede_item(), - verify_run_item(), - staleness_check_item(), - context_agent_item(), - compile_run_item(), - conflicts_list_item(), - conflicts_commit_item(), - conflict_resolve_item(), - relate_run_item(), - host_fingerprint_item(), - stats_read_item(), - knowledge_index_item(), - code_index_build_item(), - knowledge_reconcile_item(), - knowledge_triangulate_item(), - workspace_status_item(), - ] + OPERATIONS + .iter() + .map(|operation| (operation.item)()) + .collect() } -/// Register all built-in texo operations. +/// Register every operation handler. /// /// # Errors -/// Returns [`syncbat::BuildError`] if a descriptor or handler cannot be -/// registered with the builder. +/// +/// Returns the first registration error reported by syncbat. pub fn register_all(builder: &mut CoreBuilder) -> Result<(), syncbat::BuildError> { - register_workspace_init(builder)?; - register_ingest_run(builder)?; - register_claims_list(builder)?; - register_claims_search(builder)?; - register_knowledge_search(builder)?; - register_claim_explain(builder)?; - register_claim_supersede(builder)?; - register_verify_run(builder)?; - register_staleness_check(builder)?; - register_context_agent(builder)?; - register_compile_run(builder)?; - register_conflicts_list(builder)?; - register_conflicts_commit(builder)?; - register_conflict_resolve(builder)?; - register_relate_run(builder)?; - register_host_fingerprint(builder)?; - register_stats_read(builder)?; - register_knowledge_index(builder)?; - register_code_index_build(builder)?; - register_knowledge_reconcile(builder)?; - register_knowledge_triangulate(builder)?; - register_workspace_status(builder)?; - Ok(()) -} - -#[derive(Debug, Deserialize)] -struct WorkspaceInitInput { - workspace_id: String, -} - -#[derive(Debug, Deserialize)] -struct KnowledgeIndexInput { - observed_at_ms: u64, - #[serde(default)] - max_files: Option, - #[serde(default)] - max_file_bytes: Option, - #[serde(default)] - max_total_bytes: Option, -} - -impl KnowledgeIndexInput { - fn validated_limits(&self) -> Result { - let defaults = CaptureLimits::default(); - let limits = CaptureLimits { - max_files: self.max_files.unwrap_or(defaults.max_files), - max_file_bytes: self.max_file_bytes.unwrap_or(defaults.max_file_bytes), - max_total_bytes: self.max_total_bytes.unwrap_or(defaults.max_total_bytes), - }; - if limits.max_files == 0 - || limits.max_files > 100_000 - || limits.max_file_bytes == 0 - || limits.max_file_bytes > 16 * 1024 * 1024 - || limits.max_total_bytes == 0 - || limits.max_total_bytes > 512 * 1024 * 1024 - { - return Err(TexoError::OpInput { - op: "texo.knowledge.index".to_string(), - detail: "capture limits must be non-zero and at most 100000 files, 16 MiB per file, and 512 MiB total".to_string(), - }); - } - Ok(limits) - } -} - -#[derive(Debug, Serialize)] -struct KnowledgeIndexOutput { - workspace_id: String, - snapshot_id: crate::knowledge::SourceSnapshotId, - base_commit: crate::knowledge::GitObjectId, - dirty: bool, - sources_captured: usize, - evidence_recorded: usize, - claims_linked: usize, - relations_recorded: usize, - supersessions_applied: usize, - supersessions_held: usize, - held_supersessions: Vec, - already_indexed: bool, - coverage: KnowledgeCoverage, - receipts: Vec, -} - -#[derive(Debug, Deserialize)] -struct CodeIndexBuildInput { - #[serde(default)] - snapshot_id: Option, - #[serde(default)] - scip_path: Option, - observed_at_ms: u64, - #[serde(default)] - max_scip_bytes: Option, - #[serde(default)] - max_documents: Option, - #[serde(default)] - max_occurrences: Option, - #[serde(default)] - analysis_budget_secs: Option, -} - -impl CodeIndexBuildInput { - fn can_reuse_default(&self) -> bool { - self.scip_path.is_none() - && self.max_scip_bytes.is_none() - && self.max_documents.is_none() - && self.max_occurrences.is_none() - && self.analysis_budget_secs.is_none() - } - - fn validated_limits(&self) -> Result { - let defaults = CodeIndexLimits::default(); - let limits = CodeIndexLimits { - max_scip_bytes: self.max_scip_bytes.unwrap_or(defaults.max_scip_bytes), - max_documents: self.max_documents.unwrap_or(defaults.max_documents), - max_occurrences: self.max_occurrences.unwrap_or(defaults.max_occurrences), - analysis_budget: std::time::Duration::from_secs( - self.analysis_budget_secs - .unwrap_or(defaults.analysis_budget.as_secs()), - ), - }; - if limits.max_scip_bytes == 0 - || limits.max_scip_bytes > 256 * 1024 * 1024 - || limits.max_documents == 0 - || limits.max_documents > 100_000 - || limits.max_occurrences == 0 - || limits.max_occurrences > 2_000_000 - || limits.analysis_budget.is_zero() - || limits.analysis_budget > std::time::Duration::from_secs(300) - { - return Err(TexoError::OpInput { - op: "texo.code.index.build".to_string(), - detail: "code-index limits must be non-zero and at most 256 MiB, 100000 documents, 2000000 occurrences, and 300 seconds".to_string(), - }); - } - Ok(limits) - } -} - -#[derive(Debug, Serialize)] -struct CodeIndexBuildOutput { - workspace_id: String, - snapshot_id: crate::knowledge::SourceSnapshotId, - index_id: CodeIndexId, - format: crate::knowledge::CodeIndexFormat, - analyzer_fingerprint: String, - artifact_digest_hex: String, - artifact_path: String, - coverage: KnowledgeCoverage, - already_indexed: bool, - receipt: Option, -} - -#[derive(Debug, Deserialize)] -struct StatsReadInput {} - -#[derive(Debug, Deserialize)] -struct WorkspaceStatusInput { - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct StatsReadOutput { - journal_id: crate::topology::JournalId, - journal_role: crate::topology::JournalRole, - claims_total: usize, - events_total: usize, - journal_bytes: u64, - projection_bytes: u64, - agent_context_bytes: u64, - frontier_sequence: u64, -} - -#[derive(Debug, Serialize)] -struct WorkspaceStatusOutput { - workspace_id: String, - journal_id: crate::topology::JournalId, - journal_role: crate::topology::JournalRole, - source_journal: Option, - replica_mode: Option, - frontier: u64, - freshness: crate::claims::workspace::ProjectionFreshness, - claims_total: usize, - open_conflicts: usize, - settlement_complete: bool, - unresolved_pairs: usize, - authority_warnings: usize, - code_index_available: bool, - snapshot: SnapshotRead, - coverage: KnowledgeCoverage, -} - -#[derive(Debug, Serialize)] -struct WorkspaceInitOutput { - workspace_id: String, - config_path: String, - already_initialized: bool, - receipt: ReceiptNote, -} - -#[derive(Debug, Deserialize)] -struct IngestRunInput { - path: PathBuf, - dry_run: bool, - #[serde(default)] - strict: bool, - observed_at_ms: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "lowercase")] -enum IngestCompletion { - Complete, - Partial, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -enum SourceFailureCode { - #[serde(rename = "source.utf8")] - Utf8, - #[serde(rename = "source.io")] - Io, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct SourceSkipRow { - path: String, - code: SourceFailureCode, - detail: String, -} - -#[derive(Debug, Serialize)] -struct IngestRunOutput { - outcome: IngestCompletion, - workspace_id: String, - sources_observed: u32, - claims_recorded: u32, - claims_superseded: u32, - supersessions_held: usize, - held_supersessions: Vec, - dry_run: bool, - empty: bool, - skipped: Vec, - skipped_total: usize, - #[serde(skip_serializing_if = "Option::is_none")] - skipped_artifact: Option, - phase_ms: IngestPhaseMs, - events_appended: u64, - receipts: Vec, -} - -#[derive(Debug, Serialize)] -struct IngestPhaseMs { - discover: u64, - extract: u64, - append: u64, - project: u64, -} - -#[derive(Debug, Deserialize)] -struct ClaimsListInput { - subject: Option, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Deserialize)] -struct ClaimsSearchInput { - query: Option, - subject: Option, - status: Option, - limit: Option, - cursor: Option, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct ClaimsListOutput { - workspace_id: String, - frontier: u64, - claims: Vec, - snapshot: SnapshotRead, -} - -#[derive(Debug, Serialize)] -struct ClaimsSearchOutput { - workspace_id: String, - frontier: u64, - freshness: crate::claims::workspace::ProjectionFreshness, - total: usize, - returned: usize, - has_more: bool, - next_cursor: Option, - claims: Vec, - snapshot: SnapshotRead, -} - -#[derive(Debug, Deserialize)] -struct KnowledgeSearchInput { - #[serde(default)] - query: Option, - #[serde(default)] - subject: Option, - #[serde(default)] - status: Option, - #[serde(default)] - limit: Option, - #[serde(default)] - cursor: Option, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct KnowledgeSearchOutput { - workspace_id: String, - frontier: u64, - total: usize, - returned: usize, - has_more: bool, - next_cursor: Option, - results: Vec, - code_index_available: bool, - coverage: KnowledgeCoverage, - snapshot: SnapshotRead, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum KnowledgeSearchResult { - Claim { claim: AgentClaimRow }, - Code { occurrence: CodeOccurrence }, -} - -struct RankedKnowledgeResult { - rank: u8, - key: String, - result: KnowledgeSearchResult, -} - -#[derive(Debug, Deserialize)] -struct ClaimExplainInput { - claim_id: String, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct ClaimExplainOutput { - card: ClaimCard, - timeline: Vec, - answer_state: AnswerState, - evidence: Vec, - coverage: KnowledgeCoverage, - snapshot: SnapshotRead, -} - -#[derive(Debug, Deserialize)] -struct KnowledgeTriangulateInput { - target: TriangulationTarget, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct KnowledgeTriangulateOutput { - target: TriangulationTarget, - answer_state: AnswerState, - assertions: Vec, - evidence: Vec, - structural_evidence: Vec, - uncertainty: Vec, - coverage: KnowledgeCoverage, - settlement_complete: bool, - snapshot: SnapshotRead, -} - -#[derive(Debug, Deserialize)] -struct ClaimSupersedeInput { - old: String, - new: String, - reason: String, - decided_by: String, - observed_at_ms: u64, -} - -#[derive(Debug, Serialize)] -struct ClaimSupersedeOutput { - old: String, - new: String, - already_applied: bool, - receipt: Option, -} - -#[derive(Debug, Deserialize)] -struct VerifyRunInput {} - -#[derive(Debug, Serialize)] -struct VerifyRunOutput { - projection_ok: bool, - journal_ok: bool, - transitions_ok: bool, - errors: Vec, - replay_ms: u64, - events_replayed: usize, -} - -#[derive(Debug, Deserialize)] -struct StalenessCheckInput { - path: PathBuf, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct StalenessReport { - workspace_id: String, - checked_path: String, - replayed_through_sequence: u64, - diagnostics: Vec, - snapshot: SnapshotRead, -} - -#[derive(Debug, Serialize)] -struct StaleDiagnostic { - file: String, - line_start: u32, - line_end: u32, - severity: DiagnosticSeverity, - message: String, - claim_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - superseded_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - source: Option, - #[serde(skip_serializing_if = "Option::is_none")] - receipt: Option, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "lowercase")] -enum DiagnosticSeverity { - Warning, -} - -#[derive(Debug, Serialize)] -struct DiagnosticSource { - path: String, - line_start: u32, -} - -#[derive(Debug, Deserialize)] -struct ContextAgentInput { - subject: Option, - include_stale: bool, - #[serde(default)] - strict_settlement: bool, - #[serde(default)] - snapshot: Option, -} - -#[derive(Debug, Serialize)] -struct AgentContextOutput { - workspace_id: String, - replayed_through_sequence: u64, - freshness: FreshnessView, - claims: Vec, - stale_claims: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - conflicts: Vec, - snapshot: SnapshotRead, -} - -#[derive(Debug, Serialize)] -struct FreshnessView { - kind: crate::claims::workspace::ProjectionFreshness, - description: String, -} - -#[derive(Debug, Serialize)] -struct AgentStaleClaimRow { - claim_id: String, - text: String, - superseded_by: String, -} - -#[derive(Debug, Serialize)] -struct AgentConflictRow { - conflict_id: String, - claim_a: String, - claim_a_text: String, - claim_b: String, - claim_b_text: String, - reason: String, -} - -#[derive(Debug, Deserialize)] -struct CompileRunInput { - out_dir: PathBuf, - observed_at_ms: u64, - #[serde(default)] - strict_settlement: bool, -} - -#[derive(Debug, Serialize)] -struct CompileRunOutput { - files: Vec, - receipt: ReceiptNote, -} - -struct CompileFile { - name: String, - contents: String, -} - -#[derive(Debug, Deserialize)] -struct ConflictsListInput {} - -#[derive(Debug, Serialize)] -struct ConflictsOutput { - open: Vec, - resolved: Vec, -} - -#[derive(Debug, Serialize)] -struct ConflictRow { - conflict_id: String, - claim_a: String, - claim_b: String, - subject_hint: String, - reason: String, - status: crate::claims::status::ConflictStatus, -} - -#[derive(Debug, Deserialize)] -struct ConflictsCommitInput { - observed_at_ms: u64, -} - -#[derive(Debug, Serialize)] -struct CommittedConflict { - conflict_id: String, - sequence: u64, - receipt: ReceiptNote, -} - -#[derive(Debug, Deserialize)] -struct ConflictResolveInput { - conflict_id: String, - resolution: String, - resolved_by: String, - observed_at_ms: u64, -} - -#[derive(Debug, Serialize)] -struct ConflictResolveOutput { - conflict_id: String, - resolution: String, - already_applied: bool, - receipt: Option, -} - -#[derive(Debug, Deserialize)] -struct RelateRunInput { - observed_at_ms: u64, - #[serde(default)] - strict: bool, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum RelateCompletion { - Complete, - Partial, -} - -#[derive(Debug, Serialize)] -pub(crate) struct RelateRunOutput { - outcome: RelateCompletion, - pub(crate) claims_related: usize, - pub(crate) supersessions: Vec, - pub(crate) conflicts: Vec, - unresolved: Vec, - held: Vec, - warnings: Vec, - authority_warnings: Vec, - pub(crate) receipts: Vec, -} - -#[derive(Debug, Serialize)] -pub(crate) struct RelateSupersessionRow { - old_claim_id: String, - new_claim_id: String, - reason: String, - cache_key: String, -} - -#[derive(Debug, Serialize)] -pub(crate) struct RelateConflictRow { - conflict_id: String, - claim_a: String, - claim_b: String, - reason: String, - cache_key: String, -} - -#[derive(Debug, Deserialize)] -struct HostFingerprintInput {} - -#[derive(Debug, Serialize)] -struct AgentClaimRow { - claim_id: String, - status: crate::claims::status::ClaimStatus, - subject_hint: Option, - text: String, - source: AgentSourceRow, - receipt: AgentReceiptRow, - supersedes: Vec, - superseded_by: Option, -} - -#[derive(Debug, Serialize)] -struct AgentSourceRow { - source_id: String, - path: String, - line_start: u32, -} - -#[derive(Debug, Clone, Serialize)] -struct AgentReceiptRow { - event_id: String, - sequence: u64, -} - -pub(crate) struct PlannedSource { - pub(crate) observed: SourceObservedV2, - pub(crate) claims: Vec, -} - -pub(crate) struct SourcePlan { - pub(crate) sources: Vec, - pub(crate) skipped: Vec, - empty: bool, - succeeded: usize, - discover_ms: u64, - extract_ms: u64, -} - -#[derive(Debug, Deserialize)] -struct CmdClaimLine { - line_start: u32, - text: String, - normalized_text: String, - subject_hint: Option, - predicate_hint: Option, - object_hint: Option, - confidence_ppm: u32, - char_start: Option, - char_end: Option, - extractor_model: Option, - prompt_version: Option, -} - -pub(crate) fn run_op( - op: &'static str, - f: impl FnOnce() -> Result, -) -> HandlerResult { - let output = f()?; - batpak::canonical::to_bytes(&output).map_err(|error| { - HandlerError::from(TexoError::OpRuntime { - op: op.to_string(), - detail: error.to_string(), - denied: false, - }) - }) -} - -pub(crate) fn parse_input( - op: &str, - input: &[u8], -) -> Result { - batpak::canonical::from_bytes(input).map_err(|error| TexoError::OpInput { - op: op.to_string(), - detail: error.to_string(), - }) -} - -pub(crate) fn append_json( - op: &str, - cx: &mut syncbat::Ctx<'_>, - kind: EventKind, - payload: &T, -) -> Result<(), TexoError> { - let bytes = batpak::canonical::to_bytes(payload).map_err(|error| TexoError::OpRuntime { - op: op.to_string(), - detail: format!("canonical effect payload encoding failed: {error}"), - denied: false, - })?; - cx.event_append_handle() - .append_event(kind, &bytes) - .map_err(|error| op_runtime(op, error)) -} - -pub(crate) fn take_receipts() -> Result, TexoError> { - env::with(|op_env| op_env.receipts.borrow_mut().drain(..).collect()) -} - -pub(crate) fn take_one_receipt(op: &str) -> Result { - let mut receipts = take_receipts()?; - receipts.pop().ok_or_else(|| TexoError::OpRuntime { - op: op.to_string(), - detail: "append produced no receipt".to_string(), - denied: false, - }) -} - -pub(crate) fn op_runtime(op: &str, error: impl std::fmt::Display) -> TexoError { - TexoError::OpRuntime { - op: op.to_string(), - detail: error.to_string(), - denied: false, - } -} - -fn config_error(error: crate::config::ConfigError) -> TexoError { - TexoError::Config { - detail: error.to_string(), - source: Some(Box::new(error)), - } -} - -pub(crate) fn assemble_current_view() -> Result, TexoError> { - env::with(|op_env| { - let mut cache = op_env.cache.borrow_mut(); - assemble(&op_env.store, &op_env.workspace_id, &mut cache) - })? -} - -fn assemble_snapshot_view( - requested: Option<&str>, -) -> Result<(std::sync::Arc, SnapshotRead), TexoError> { - let Some(requested) = requested else { - let view = assemble_current_view()?; - let snapshot = snapshot_for_view(&view)?; - return Ok((view, snapshot)); - }; - let (store, workspace, journal_id) = env::with(|op_env| { - ( - op_env.store.clone(), - op_env.workspace_id.clone(), - op_env.journal.id.clone(), - ) - })?; - let workspace_id = WorkspaceId::new(workspace.clone())?; - let descriptor = SnapshotToken::resolve_for_journal(requested, &workspace_id, &journal_id) - .map_err(|error| TexoError::Snapshot { - kind: SnapshotFailureKind::InvalidToken, - detail: error.to_string(), - })?; - let available_source = - latest_source_snapshot(Some(descriptor.frontier))?.map(|snapshot| snapshot.snapshot_id); - if available_source != descriptor.source_snapshot_id { - return Err(TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "the token's source snapshot is unavailable at its journal frontier" - .to_string(), - }); - } - validate_snapshot_anchor(&store, &workspace, &descriptor)?; - let view = assemble_through(&store, &workspace, descriptor.frontier).map_err(|error| { - TexoError::Snapshot { - kind: SnapshotFailureKind::Unavailable, - detail: error.to_string(), - } - })?; - Ok((view, SnapshotRead::new(descriptor))) -} - -fn snapshot_for_view(view: &WorkspaceView) -> Result { - let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; - let (anchor_event_id_hex, journal_id) = env::with(|op_env| { - Ok::<_, TexoError>(( - anchor_at_frontier(&op_env.store, &op_env.workspace_id, view.frontier)?, - op_env.journal.id.clone(), - )) - })??; - let source_snapshot_id = - latest_source_snapshot(Some(view.frontier))?.map(|snapshot| snapshot.snapshot_id); - Ok(SnapshotRead::new(SnapshotDescriptor { - workspace_id, - journal_id, - frontier: view.frontier, - anchor_event_id_hex, - source_snapshot_id, - })) -} - -fn validate_snapshot_anchor( - store: &crate::journal_store::JournalStore, - workspace_id: &str, - descriptor: &SnapshotDescriptor, -) -> Result<(), TexoError> { - let actual = anchor_at_frontier(store, workspace_id, descriptor.frontier)?; - if actual != descriptor.anchor_event_id_hex { - return Err(TexoError::Snapshot { - kind: SnapshotFailureKind::AnchorMismatch, - detail: format!( - "journal anchor at frontier {} differs from the token", - descriptor.frontier - ), - }); + for operation in OPERATIONS { + (operation.register)(builder)?; } Ok(()) } - -fn anchor_at_frontier( - store: &crate::journal_store::JournalStore, - workspace_id: &str, - frontier: u64, -) -> Result { - if frontier == 0 { - return Ok(String::new()); - } - let region = Region::scope(scope_for_workspace(workspace_id)); - let entry = store - .query_entries_after(®ion, Some(frontier.saturating_sub(1)), 1) - .into_iter() - .next() - .filter(|entry| entry.global_sequence() == frontier) - .ok_or_else(|| TexoError::Snapshot { - kind: SnapshotFailureKind::Unavailable, - detail: format!("workspace frontier {frontier} is unavailable"), - })?; - Ok(format!("{:032x}", entry.event_id().as_u128())) -} - -fn claim_timeline_through(entity: &str, frontier: u64) -> Result { - env::with(|op_env| { - let mut timeline = ClaimTimeline::default(); - for entry in op_env.store.by_entity(entity) { - if entry.global_sequence() > frontier { - break; - } - let raw = op_env.store.read_raw(entry.event_id())?; - timeline.apply_event(&raw.event); - } - Ok::<_, TexoError>(timeline) - })? -} - -fn coverage_for_view( - view: &WorkspaceView, - snapshot: &SnapshotRead, -) -> Result { - // Only a genuinely missing snapshot (`Ok(None)`) or a mismatched identity - // degrades to `Unavailable`; a decode/corruption error must bubble up rather - // than masquerade as an ordinary absent snapshot. - if let Some(recorded) = latest_source_snapshot(Some(view.frontier))? { - if Some(&recorded.snapshot_id) == snapshot.descriptor.source_snapshot_id.as_ref() { - return Ok(recorded.coverage); - } - } - Ok(KnowledgeCoverage { - analysis_quality: AnalysisQuality::Unavailable, - sources_examined: u64::try_from(view.sources.len()).unwrap_or(u64::MAX), - occurrences: u64::try_from(view.claims.len()).unwrap_or(u64::MAX), - truncated: false, - gaps: vec![CoverageGap { - path: None, - kind: CoverageGapKind::SourceSnapshotUnavailable, - }], - }) -} - -fn status_coverage( - view: &WorkspaceView, - snapshot: &SnapshotRead, -) -> Result<(KnowledgeCoverage, bool), TexoError> { - let mut coverage = coverage_for_view(view, snapshot)?; - let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() else { - return Ok((coverage, false)); - }; - let loaded = load_code_artifact_at(view.frontier, source_snapshot_id)?; - if let Some(code_coverage) = loaded.coverage { - merge_coverage(&mut coverage, &code_coverage); - } - if loaded.unavailable { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::CodeIndexUnavailable, - }); - } - Ok((coverage, !loaded.unavailable)) -} - -/// Reconstruct the capture bounds a snapshot was recorded with so code indexing -/// recaptures the same bounded world instead of the defaults. -fn recorded_capture_limits(recorded: &SourceSnapshotRecordedV1) -> CaptureLimits { - CaptureLimits { - max_files: usize::try_from(recorded.capture_max_files).unwrap_or(usize::MAX), - max_file_bytes: recorded.capture_max_file_bytes, - max_total_bytes: recorded.capture_max_total_bytes, - } -} - -fn latest_source_snapshot( - frontier: Option, -) -> Result, TexoError> { - Ok(source_snapshots_through(frontier)?.pop()) -} - -/// The recorded snapshot with this content-addressed id, regardless of whether a -/// newer snapshot was recorded afterward (revisiting a prior commit re-derives an -/// existing id rather than appending a new record). -fn source_snapshot_by_id( - snapshot_id: &crate::knowledge::SourceSnapshotId, -) -> Result, TexoError> { - Ok(source_snapshots_through(None)? - .into_iter() - .rev() - .find(|snapshot| &snapshot.snapshot_id == snapshot_id)) -} - -fn source_snapshots_through( - frontier: Option, -) -> Result, TexoError> { - env::with(|op_env| { - let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); - let mut after = None; - let mut snapshots = Vec::new(); - 'pages: loop { - let page = op_env.store.query_entries_after(®ion, after, 256); - if page.is_empty() { - break; - } - for entry in &page { - if frontier.is_some_and(|frontier| entry.global_sequence() > frontier) { - break 'pages; - } - if entry.event_kind() == ::KIND { - let raw = op_env.store.read_raw(entry.event_id())?; - snapshots.push( - batpak::encoding::from_bytes::( - &raw.event.payload, - ) - .map_err(|error| TexoError::Decode { - entity: entry.coord().entity().to_string(), - detail: error.to_string(), - })?, - ); - } - } - after = page.last().map(batpak::store::IndexEntry::global_sequence); - } - Ok::<_, TexoError>(snapshots) - })? -} - -fn plan_snapshot_relations( - root: &Path, - workspace_id: &WorkspaceId, - capture: &crate::git_source::GitCapture, - previous: &[SourceSnapshotRecordedV1], - observed_at_ms: u64, -) -> Result<(Vec, Vec), TexoError> { - let skipped = previous - .len() - .saturating_sub(MAX_TEMPORAL_SNAPSHOT_COMPARISONS); - let mut gaps = Vec::new(); - if skipped > 0 { - gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::BudgetExceeded, - }); - } - let mut relations = Vec::new(); - for prior in previous.iter().skip(skipped) { - if prior.snapshot_id == capture.snapshot_id { - continue; - } - let comparison = if prior.repository_id == capture.repository_id { - overlay_aware_comparison(root, prior, capture)? - } else { - crate::git_source::GitComparison { - relation: TemporalRelation::Unknown, - gap: Some(CoverageGapKind::MissingObject), - } - }; - if let Some(kind) = comparison.gap { - let gap = CoverageGap { path: None, kind }; - if !gaps.contains(&gap) { - gaps.push(gap); - } - } - // Never journal an `Unknown` ordering: the relation idempotency key is - // (workspace, left, right) and replay keeps the first fact, so a durable - // Unknown from shallow history or an exhausted walk would permanently - // shadow the real Before/After discoverable once full history arrives. - // Its absence already reads as Unknown, and the gap above records why. - if comparison.relation == TemporalRelation::Unknown { - continue; - } - relations.push(SourceSnapshotRelationV1 { - workspace_id: workspace_id.clone(), - repository_id: capture.repository_id.clone(), - left_snapshot_id: prior.snapshot_id.clone(), - right_snapshot_id: capture.snapshot_id.clone(), - left_commit: prior.base_commit.clone(), - right_commit: capture.base_commit.clone(), - relation: comparison.relation, - observed_at_ms, - }); - } - Ok((relations, gaps)) -} - -fn overlay_aware_comparison( - root: &Path, - prior: &SourceSnapshotRecordedV1, - capture: &crate::git_source::GitCapture, -) -> Result { - use crate::git_source::GitComparison; - - if prior.base_commit == capture.base_commit { - return Ok(GitComparison { - relation: match (prior.dirty, capture.dirty) { - (false, true) => TemporalRelation::Before, - (true, false) => TemporalRelation::After, - (true, true) => TemporalRelation::Concurrent, - (false, false) => TemporalRelation::Same, - }, - gap: None, - }); - } - let comparison = compare_commits( - root, - &prior.base_commit, - &capture.base_commit, - MAX_GIT_ANCESTRY_WALK, - )?; - let relation = match comparison.relation { - TemporalRelation::Before if prior.dirty => TemporalRelation::Concurrent, - TemporalRelation::After if capture.dirty => TemporalRelation::Concurrent, - TemporalRelation::Same => TemporalRelation::Same, - TemporalRelation::Before => TemporalRelation::Before, - TemporalRelation::After => TemporalRelation::After, - TemporalRelation::Concurrent => TemporalRelation::Concurrent, - TemporalRelation::Unknown => TemporalRelation::Unknown, - }; - Ok(GitComparison { - relation, - gap: comparison.gap, - }) -} - -fn plan_and_attach_snapshot_relations( - root: &Path, - workspace_id: &WorkspaceId, - capture: &mut crate::git_source::GitCapture, - previous: &[SourceSnapshotRecordedV1], - observed_at_ms: u64, -) -> Result, TexoError> { - let (relations, gaps) = - plan_snapshot_relations(root, workspace_id, capture, previous, observed_at_ms)?; - for gap in gaps { - if capture.coverage.gaps.len() < 256 && !capture.coverage.gaps.contains(&gap) { - capture.coverage.gaps.push(gap); - } - } - Ok(relations) -} - -fn evidence_projection_through(frontier: u64) -> Result { - env::with(|op_env| assemble_evidence_through(&op_env.store, &op_env.workspace_id, frontier))? -} - -fn temporal_projection_through(frontier: u64) -> Result { - env::with(|op_env| assemble_temporal_through(&op_env.store, &op_env.workspace_id, frontier))? -} - -pub(crate) fn workspace_temporal_policy( - view: &WorkspaceView, -) -> Result { - workspace_temporal_policy_through(view, view.frontier) -} - -fn workspace_temporal_policy_through( - view: &WorkspaceView, - frontier: u64, -) -> Result { - let evidence = evidence_projection_through(frontier)?; - let relations = temporal_projection_through(frontier)?; - let mut policy = RelateTemporalPolicy::default(); - for claim in &view.claims { - let claim_id = ClaimId::try_from(claim.card.claim_id.as_str())?; - if let Some(latest) = evidence - .for_claim(claim_id.as_str()) - .iter() - .filter(|item| { - item.method == EvidenceLinkMethod::Deterministic - && item.stance == EvidenceStance::Supports - }) - .max_by_key(|item| item.link_sequence) - { - policy.bind_claim(&claim_id, &latest.occurrence.snapshot_id); - } - } - for (left, right, relation) in relations.facts() { - policy.insert_relation_ids(left, right, relation); - } - Ok(policy) -} - -fn semantic_temporal_policy(view: &WorkspaceView) -> Result { - workspace_temporal_policy(view) -} - -fn triangulate_from_view( - view: &WorkspaceView, - snapshot: &SnapshotRead, - target: TriangulationTarget, -) -> Result { - validate_triangulation_target(&target)?; - let projection = evidence_projection_through(view.frontier)?; - let claim_ids = triangulation_claim_ids(view, &target)?; - let assertions = claim_list_rows(view, None)? - .into_iter() - .filter(|claim| claim_ids.contains(&claim.claim_id)) - .collect::>(); - let mut evidence = claim_ids - .iter() - .flat_map(|claim_id| projection.for_claim(claim_id).iter().cloned()) - .collect::>(); - evidence.retain(|item| evidence_matches_target(item, &target)); - let mut coverage = coverage_for_view(view, snapshot)?; - let code = code_evidence_for_target(view.frontier, snapshot, &target)?; - if let Some(code_coverage) = &code.coverage { - merge_coverage(&mut coverage, code_coverage); - } - if projection.is_incomplete() { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::AnalysisIncomplete, - }); - } - let settlement = authoritative_settlements(Some(view.frontier))?; - let settlement_complete = settlement.unresolved_pairs == 0; - let mut uncertainty = BTreeSet::new(); - if snapshot.descriptor.source_snapshot_id.is_none() { - uncertainty.insert(UncertaintyReason::SourceSnapshotUnavailable); - } - if coverage.truncated || !coverage.gaps.is_empty() { - uncertainty.insert(UncertaintyReason::PartialCoverage); - } - if !settlement_complete { - uncertainty.insert(UncertaintyReason::SettlementIncomplete); - } - if matches!(target, TriangulationTarget::Symbol { .. }) && code.unavailable { - uncertainty.insert(UncertaintyReason::CodeIndexUnavailable); - if !coverage - .gaps - .iter() - .any(|gap| gap.kind == CoverageGapKind::CodeIndexUnavailable) - { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::CodeIndexUnavailable, - }); - } - } - if !assertions.is_empty() && evidence.is_empty() { - uncertainty.insert(UncertaintyReason::ExactEvidenceUnavailable); - } - let answer_state = answer_state_for_rows(&assertions, &evidence, &code.rows); - Ok(KnowledgeTriangulateOutput { - target, - answer_state, - assertions, - evidence, - structural_evidence: code.rows, - uncertainty: uncertainty.into_iter().collect(), - coverage, - settlement_complete, - snapshot: snapshot.clone(), - }) -} - -#[derive(Default)] -struct CodeEvidenceLookup { - rows: Vec, - coverage: Option, - unavailable: bool, -} - -#[derive(Default)] -struct LoadedCodeArtifact { - artifact: Option, - coverage: Option, - unavailable: bool, -} - -fn code_evidence_for_target( - frontier: u64, - snapshot: &SnapshotRead, - target: &TriangulationTarget, -) -> Result { - let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() else { - return Ok(CodeEvidenceLookup { - unavailable: true, - ..CodeEvidenceLookup::default() - }); - }; - let loaded = load_code_artifact_at(frontier, source_snapshot_id)?; - let Some(artifact) = loaded.artifact else { - return Ok(CodeEvidenceLookup { - coverage: loaded.coverage, - unavailable: loaded.unavailable, - ..CodeEvidenceLookup::default() - }); - }; - let mut rows = artifact - .occurrences - .into_iter() - .filter(|occurrence| code_occurrence_matches(occurrence, target)) - .take(MAX_TRIANGULATION_CODE_OCCURRENCES + 1) - .collect::>(); - let mut coverage = artifact.coverage; - if rows.len() > MAX_TRIANGULATION_CODE_OCCURRENCES { - rows.truncate(MAX_TRIANGULATION_CODE_OCCURRENCES); - coverage.truncated = true; - if !coverage - .gaps - .iter() - .any(|gap| gap.path.is_none() && gap.kind == CoverageGapKind::BudgetExceeded) - { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::BudgetExceeded, - }); - } - } - Ok(CodeEvidenceLookup { - rows, - coverage: Some(coverage), - unavailable: false, - }) -} - -fn load_code_artifact_at( - frontier: u64, - source_snapshot_id: &crate::knowledge::SourceSnapshotId, -) -> Result { - let Some(recorded) = latest_code_index(Some(frontier), source_snapshot_id)? else { - return Ok(LoadedCodeArtifact { - unavailable: true, - ..LoadedCodeArtifact::default() - }); - }; - let artifact = env::with(|op_env| { - load_code_index( - &op_env.root, - &recorded.index_id, - &recorded.artifact_digest_hex, - ) - })??; - if artifact - .as_ref() - .is_some_and(|artifact| artifact.snapshot_id != *source_snapshot_id) - { - return Err(TexoError::Snapshot { - kind: SnapshotFailureKind::SourceUnavailable, - detail: "code-index artifact belongs to a different source snapshot".to_string(), - }); - } - Ok(LoadedCodeArtifact { - unavailable: artifact.is_none(), - coverage: Some(recorded.coverage), - artifact, - }) -} - -fn latest_code_index( - frontier: Option, - snapshot_id: &crate::knowledge::SourceSnapshotId, -) -> Result, TexoError> { - env::with(|op_env| { - let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); - let mut after = None; - let mut latest = None; - 'pages: loop { - let page = op_env.store.query_entries_after(®ion, after, 256); - if page.is_empty() { - break; - } - for entry in &page { - if frontier.is_some_and(|frontier| entry.global_sequence() > frontier) { - break 'pages; - } - if entry.event_kind() == ::KIND { - let raw = op_env.store.read_raw(entry.event_id())?; - let payload = - batpak::encoding::from_bytes::(&raw.event.payload) - .map_err(|error| TexoError::Decode { - entity: entry.coord().entity().to_string(), - detail: error.to_string(), - })?; - if payload.snapshot_id == *snapshot_id { - latest = Some(payload); - } - } - } - after = page.last().map(batpak::store::IndexEntry::global_sequence); - } - Ok::<_, TexoError>(latest) - })? -} - -fn code_occurrence_matches(occurrence: &CodeOccurrence, target: &TriangulationTarget) -> bool { - match target { - TriangulationTarget::Claim { .. } => false, - TriangulationTarget::Path { - path, - line_start, - line_end, - } => { - occurrence.path == *path - && line_start.is_none_or(|start| occurrence.line_range.end >= start) - && line_end.is_none_or(|end| occurrence.line_range.start <= end) - } - TriangulationTarget::Symbol { symbol } => { - occurrence.symbol == *symbol || occurrence.display_name == *symbol - } - } -} - -fn merge_coverage(target: &mut KnowledgeCoverage, code: &KnowledgeCoverage) { - if analysis_quality_rank(code.analysis_quality) > analysis_quality_rank(target.analysis_quality) - { - target.analysis_quality = code.analysis_quality; - } - target.sources_examined = target.sources_examined.max(code.sources_examined); - target.occurrences = target.occurrences.saturating_add(code.occurrences); - target.truncated |= code.truncated; - for gap in &code.gaps { - if target.gaps.len() >= 256 { - target.truncated = true; - break; - } - if !target.gaps.contains(gap) { - target.gaps.push(gap.clone()); - } - } -} - -const fn analysis_quality_rank(quality: AnalysisQuality) -> u8 { - match quality { - AnalysisQuality::Precise => 3, - AnalysisQuality::Syntactic => 2, - AnalysisQuality::Lexical => 1, - AnalysisQuality::Unavailable => 0, - } -} - -fn validate_triangulation_target(target: &TriangulationTarget) -> Result<(), TexoError> { - match target { - TriangulationTarget::Claim { claim_id } if claim_id.is_empty() => Err(TexoError::OpInput { - op: "texo.knowledge.triangulate".to_string(), - detail: "claim_id must not be empty".to_string(), - }), - TriangulationTarget::Path { - path, - line_start, - line_end, - } => { - let safe = !path.is_empty() - && !Path::new(path).is_absolute() - && Path::new(path) - .components() - .all(|component| matches!(component, std::path::Component::Normal(_))); - let valid_range = match (*line_start, *line_end) { - (None, None) => true, - (Some(start), Some(end)) => start > 0 && start <= end, - _ => false, - }; - if safe && valid_range { - Ok(()) - } else { - Err(TexoError::OpInput { - op: "texo.knowledge.triangulate".to_string(), - detail: "path must be repository-relative and line bounds must be absent or an ordered one-based pair".to_string(), - }) - } - } - TriangulationTarget::Symbol { symbol } if symbol.is_empty() || symbol.len() > 1024 => { - Err(TexoError::OpInput { - op: "texo.knowledge.triangulate".to_string(), - detail: "symbol must contain between 1 and 1024 bytes".to_string(), - }) - } - TriangulationTarget::Claim { .. } | TriangulationTarget::Symbol { .. } => Ok(()), - } -} - -fn triangulation_claim_ids( - view: &WorkspaceView, - target: &TriangulationTarget, -) -> Result, TexoError> { - match target { - TriangulationTarget::Claim { claim_id } => { - if view - .claims - .iter() - .any(|claim| claim.card.claim_id == *claim_id) - { - Ok(BTreeSet::from([claim_id.clone()])) - } else { - Err(TexoError::MissingEntity { - entity: entity_for_claim(claim_id), - }) - } - } - TriangulationTarget::Path { - path, - line_start, - line_end, - } => Ok(view - .claims - .iter() - .filter(|claim| claim.card.source_path == *path) - .filter(|claim| { - line_start.is_none_or(|start| claim.card.line_end >= start) - && line_end.is_none_or(|end| claim.card.line_start <= end) - }) - .map(|claim| claim.card.claim_id.clone()) - .collect()), - TriangulationTarget::Symbol { .. } => Ok(BTreeSet::new()), - } -} - -fn evidence_matches_target(evidence: &ClaimEvidence, target: &TriangulationTarget) -> bool { - match target { - TriangulationTarget::Claim { .. } => true, - TriangulationTarget::Path { - path, - line_start, - line_end, - } => { - evidence.occurrence.path == *path - && line_start.is_none_or(|start| evidence.occurrence.line_range.end >= start) - && line_end.is_none_or(|end| evidence.occurrence.line_range.start <= end) - } - TriangulationTarget::Symbol { .. } => false, - } -} - -fn answer_state_for_rows( - assertions: &[AgentClaimRow], - evidence: &[ClaimEvidence], - structural_evidence: &[CodeOccurrence], -) -> AnswerState { - use crate::claims::status::ClaimStatus; - if evidence - .iter() - .any(|item| item.stance == EvidenceStance::Contradicts) - { - AnswerState::Contradicted - } else if assertions - .iter() - .any(|claim| claim.status == ClaimStatus::Conflicting) - { - AnswerState::Incomparable - } else if assertions - .iter() - .any(|claim| claim.status == ClaimStatus::Superseded) - { - AnswerState::Stale - } else if (!assertions.is_empty() - && evidence - .iter() - .any(|item| item.stance == EvidenceStance::Supports)) - || !structural_evidence.is_empty() - { - AnswerState::Supported - } else { - AnswerState::Unverified - } -} - -fn answer_state_for_claim( - status: Option, - evidence: &[ClaimEvidence], -) -> AnswerState { - use crate::claims::status::ClaimStatus; - match status { - Some(ClaimStatus::Superseded) => AnswerState::Stale, - Some(ClaimStatus::Conflicting) => AnswerState::Incomparable, - Some(ClaimStatus::Current) - if evidence - .iter() - .any(|item| item.stance == EvidenceStance::Contradicts) => - { - AnswerState::Contradicted - } - Some(ClaimStatus::Current) - if evidence - .iter() - .any(|item| item.stance == EvidenceStance::Supports) => - { - AnswerState::Supported - } - Some(ClaimStatus::Current) | None => AnswerState::Unverified, - } -} - -struct EvidencePlan { - rows: Vec<(EvidenceOccurrence, ClaimEvidenceLinkedV1)>, - gaps: Vec, -} - -fn append_knowledge_plan( - cx: &mut syncbat::Ctx<'_>, - workspace_id: &WorkspaceId, - snapshot: &SourceSnapshotRecordedV1, - rows: &[(EvidenceOccurrence, ClaimEvidenceLinkedV1)], - relations: &[SourceSnapshotRelationV1], - observed_at_ms: u64, -) -> Result, TexoError> { - append_json( - "texo.knowledge.index", - cx, - ::KIND, - snapshot, - )?; - for (occurrence, link) in rows { - append_json( - "texo.knowledge.index", - cx, - ::KIND, - &EvidenceOccurrenceRecordedV1 { - workspace_id: workspace_id.clone(), - occurrence: occurrence.clone(), - observed_at_ms, - }, - )?; - append_json( - "texo.knowledge.index", - cx, - ::KIND, - link, - )?; - } - for relation in relations { - append_json( - "texo.knowledge.index", - cx, - ::KIND, - relation, - )?; - } - take_receipts() -} - -fn plan_claim_evidence( - view: &WorkspaceView, - sources: &[CapturedSource], - snapshot_id: &crate::knowledge::SourceSnapshotId, - observed_at_ms: u64, -) -> Result { - let by_path = sources - .iter() - .map(|source| (source.path.as_str(), source)) - .collect::>(); - let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; - let mut rows = Vec::new(); - let mut gaps = Vec::new(); - for claim in &view.claims { - let Some(source) = by_path.get(claim.card.source_path.as_str()) else { - continue; - }; - let source_digest_hex = crate::events::ids::blake3_bytes_hex(&source.bytes); - let captured_source_id = crate::events::ids::source_id_from_hash(&source_digest_hex)?; - if captured_source_id.as_str() != claim.card.source_id { - gaps.push(CoverageGap { - path: Some(source.path.clone()), - kind: CoverageGapKind::AnalysisIncomplete, - }); - continue; - } - let Some((start, end)) = - line_byte_range(&source.bytes, claim.card.line_start, claim.card.line_end) - else { - gaps.push(CoverageGap { - path: Some(source.path.clone()), - kind: CoverageGapKind::AnalysisIncomplete, - }); - continue; - }; - let excerpt_bytes = &source.bytes[start..end]; - let Ok(excerpt) = std::str::from_utf8(excerpt_bytes) else { - gaps.push(CoverageGap { - path: Some(source.path.clone()), - kind: CoverageGapKind::UnsupportedEncoding, - }); - continue; - }; - if excerpt.len() > MAX_EVIDENCE_EXCERPT_BYTES { - gaps.push(CoverageGap { - path: Some(source.path.clone()), - kind: CoverageGapKind::SourceTooLarge, - }); - continue; - } - let material = format!( - "texo.evidence.occurrence.v1\u{1f}{snapshot_id}\u{1f}{}\u{1f}{start}\u{1f}{end}\u{1f}{}", - source.path, claim.card.claim_id - ); - let occurrence_id = EvidenceOccurrenceId::derive(&material); - let occurrence = EvidenceOccurrence { - occurrence_id: occurrence_id.clone(), - snapshot_id: snapshot_id.clone(), - source_kind: match source.layer { - CapturedLayer::Committed => EvidenceSourceKind::GitBlob, - CapturedLayer::Worktree => EvidenceSourceKind::WorktreeOverlay, - }, - path: source.path.clone(), - byte_range: ByteRange::new( - u64::try_from(start).unwrap_or(u64::MAX), - u64::try_from(end).unwrap_or(u64::MAX), - ) - .map_err(|error| TexoError::Source { - path: source.path.clone(), - detail: error.to_string(), - })?, - line_range: LineRange::new(claim.card.line_start, claim.card.line_end).map_err( - |error| TexoError::Source { - path: source.path.clone(), - detail: error.to_string(), - }, - )?, - git_blob: source.blob_id.clone(), - source_digest_hex, - excerpt: excerpt.to_string(), - analyzer_fingerprint: format!( - "{}:{}:{}", - claim.card.extractor_kind, claim.card.extractor_model, claim.card.prompt_version - ), - analysis_quality: AnalysisQuality::Syntactic, - }; - occurrence.validate().map_err(|error| TexoError::Source { - path: source.path.clone(), - detail: error.to_string(), - })?; - let link = ClaimEvidenceLinkedV1 { - workspace_id: workspace_id.clone(), - claim_id: ClaimId::try_from(claim.card.claim_id.as_str())?, - occurrence_id, - stance: EvidenceStance::Supports, - method: EvidenceLinkMethod::Deterministic, - observed_at_ms, - }; - rows.push((occurrence, link)); - } - Ok(EvidencePlan { rows, gaps }) -} - -fn line_byte_range(bytes: &[u8], start_line: u32, end_line: u32) -> Option<(usize, usize)> { - if start_line == 0 || end_line < start_line { - return None; - } - let mut line = 1_u32; - let mut line_start = 0_usize; - let mut range_start = None; - for offset in 0..=bytes.len() { - let boundary = offset == bytes.len() || bytes.get(offset) == Some(&b'\n'); - if !boundary { - continue; - } - if line == start_line { - range_start = Some(line_start); - } - if line == end_line { - return range_start.map(|start| (start, offset)); - } - line = line.saturating_add(1); - line_start = offset.saturating_add(1); - } - None -} - -struct SettlementAuthority { - verdicts: BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - cache_keys: BTreeMap<(String, String), String>, - warnings: Vec, - unresolved_pairs: usize, -} - -fn authoritative_settlements(frontier: Option) -> Result { - env::with(|op_env| { - let scope = scope_for_workspace(&op_env.workspace_id); - let region = Region::scope(&scope); - let mut after = None; - let mut entities = BTreeSet::new(); - loop { - let page = op_env.store.query_entries_after(®ion, after, 256); - if page.is_empty() { - break; - } - for entry in &page { - if frontier.is_some_and(|frontier| entry.global_sequence() > frontier) { - break; - } - let entity = entry.coord().entity(); - if entity.starts_with("relation:") { - entities.insert(entity.to_string()); - } - } - if page - .last() - .is_some_and(|entry| frontier.is_some_and(|value| entry.global_sequence() > value)) - { - break; - } - after = page.last().map(batpak::store::IndexEntry::global_sequence); - } - - let mut settled = BTreeMap::new(); - let mut cache_keys = BTreeMap::new(); - let mut warnings = Vec::new(); - let mut unresolved_pairs = 0; - for entity in entities { - let card = if let Some(frontier) = frontier { - let mut card = crate::claims::settlement::SettlementCard::default(); - for entry in op_env.store.by_entity(&entity) { - if entry.global_sequence() > frontier { - break; - } - let raw = op_env.store.read_raw(entry.event_id())?; - card.apply_event(&raw.event); - } - card - } else { - let Some(card) = op_env - .store - .project::( - &entity, - &Freshness::Consistent, - )? - else { - continue; - }; - card - }; - let Some(authoritative) = card.authoritative.as_ref() else { - if !card.deferrals.is_empty() { - unresolved_pairs += 1; - } - continue; - }; - let older = ClaimId::try_from(card.older_claim.as_str())?; - let newer = ClaimId::try_from(card.newer_claim.as_str())?; - for later in &card.later_judgments { - if later.relation != authoritative.relation { - warnings.push(crate::relate::settlement::AuthorityWarning { - old_claim: older.clone(), - new_claim: newer.clone(), - prior_verdict: authoritative.relation, - prior_fingerprint: authoritative.judge_fingerprint.clone(), - new_verdict: later.relation, - new_fingerprint: later.judge_fingerprint.clone(), - message: "authoritative verdict unchanged".to_string(), - }); - } - } - settled.insert( - (older.clone(), newer.clone()), - crate::semantics::RelationVerdict { - relation: authoritative.relation.into(), - score: ppm_to_score(authoritative.score_ppm), - }, - ); - cache_keys.insert( - (older.to_string(), newer.to_string()), - authoritative.cache_key_hex.clone(), - ); - } - Ok::<_, TexoError>(SettlementAuthority { - verdicts: settled, - cache_keys, - warnings, - unresolved_pairs, - }) - })? -} - -fn require_complete_settlement() -> Result<(), TexoError> { - let unresolved = authoritative_settlements(None)?.unresolved_pairs; - if unresolved == 0 { - return Ok(()); - } - Err(TexoError::Semantics { - backend: "settlement".to_string(), - detail: format!( - "strict settlement refused authority-bearing output: {unresolved} unresolved relation pair(s); run `texo relate` to resume" - ), - }) -} - -#[expect( - clippy::cast_precision_loss, - reason = "ppm values are bounded to one million and exactly adequate for model confidence" -)] -fn ppm_to_score(score_ppm: u32) -> f32 { - score_ppm as f32 / 1_000_000.0 -} - -pub(crate) fn resolve_path(root: &Path, path: &Path) -> PathBuf { - if path.is_absolute() { - path.to_path_buf() - } else { - root.join(path) - } -} - -pub(crate) fn plan_sources( - op: &str, - root: &Path, - input_path: &Path, - workspace_id: &str, - observed_at_ms: u64, - extractor_cmd: Option<&str>, - view: &WorkspaceView, -) -> Result { - let existing_hashes = view - .sources - .iter() - .map(|source| source.body_hash_hex.clone()) - .collect::>(); - let mut batch_hashes = BTreeSet::new(); - let mut planned = Vec::new(); - let discover_started = Instant::now(); - let discovery = collect_markdown_files(input_path).map_err(|error| TexoError::Source { - path: input_path.to_string_lossy().to_string(), - detail: error.to_string(), - })?; - let discover_ms = elapsed_ms(discover_started); - let empty = discovery.files.is_empty(); - let mut skipped = discovery - .failures - .into_iter() - .map(|failure| SourceSkipRow { - path: failure.path.to_string_lossy().to_string(), - code: SourceFailureCode::Io, - detail: bounded_source_detail(&failure.error.to_string()), - }) - .collect::>(); - let mut succeeded = 0; - let extract_started = Instant::now(); - for path in discovery.files { - let doc = match MarkdownDocument::from_path(&path, root) { - Ok(doc) => doc, - Err(error) => { - skipped.push(SourceSkipRow { - path: path.to_string_lossy().to_string(), - code: match error { - crate::extract::markdown::SourceError::Utf8(_) => SourceFailureCode::Utf8, - crate::extract::markdown::SourceError::Io(_) - | crate::extract::markdown::SourceError::Walk(_) - | crate::extract::markdown::SourceError::Id(_) => SourceFailureCode::Io, - }, - detail: bounded_source_detail(&error.to_string()), - }); - continue; - } - }; - succeeded += 1; - if existing_hashes.contains(&doc.body_hash_hex) - || !batch_hashes.insert(doc.body_hash_hex.clone()) - { - continue; - } - let claims = if let Some(cmd) = extractor_cmd { - extract_cmd_claims(op, root, cmd, &path, &doc, workspace_id, observed_at_ms)? - } else { - extract_heuristic_claims(&doc, workspace_id, observed_at_ms)? - }; - planned.push(PlannedSource { - observed: SourceObservedV2 { - source_id: doc.source_id, - workspace_id: workspace_id.to_string(), - source_kind: "markdown".to_string(), - path: doc.path, - body_hash_hex: doc.body_hash_hex, - observed_at_ms, - }, - claims, - }); - } - skipped.sort_by(|left, right| left.path.cmp(&right.path)); - Ok(SourcePlan { - sources: planned, - skipped, - empty, - succeeded, - discover_ms, - extract_ms: elapsed_ms(extract_started), - }) -} - -fn elapsed_ms(started: Instant) -> u64 { - u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) -} - -fn bounded_source_detail(detail: &str) -> String { - detail - .chars() - .take(MAX_SOURCE_FAILURE_DETAIL_CHARS) - .collect() -} - -fn settle_source_failures( - root: &Path, - observed_at_ms: u64, - rows: Vec, -) -> Result<(Vec, Option), TexoError> { - if rows.len() <= MAX_INLINE_SOURCE_FAILURES { - return Ok((rows, None)); - } - let bytes = serde_json::to_vec(&rows)?; - let digest = blake3::hash(&bytes).to_hex().to_string(); - let short_digest: String = digest.chars().take(16).collect(); - let relative = PathBuf::from(".texo") - .join("operations") - .join("ingest-skips") - .join(format!("{observed_at_ms}-{short_digest}.json")); - let path = root.join(&relative); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - let temporary = path.with_extension("json.tmp"); - std::fs::write(&temporary, bytes)?; - std::fs::rename(temporary, path)?; - Ok(( - rows.into_iter().take(MAX_INLINE_SOURCE_FAILURES).collect(), - Some(relative.to_string_lossy().to_string()), - )) -} - -/// Run the semantic relate pass and journal resulting v2 transition payloads. -/// -/// # Errors -/// -/// Returns [`TexoError::Semantics`] when the configured semantic backends fail, -/// [`TexoError::Store`] when projection/event reads fail, and -/// [`TexoError::OpRuntime`] when append effects fail or no receipt is produced. -#[expect( - clippy::too_many_lines, - reason = "WO-5 keeps relate orchestration in one op chokepoint" -)] -pub(crate) fn run_relate_pass( - op: &'static str, - cx: &mut syncbat::Ctx<'_>, - observed_at_ms: u64, - strict: bool, -) -> Result { - let view = assemble_current_view()?; - let claims = semantic_claims_from_view(&view)?; - if claims.len() < 2 { - return Ok(RelateRunOutput { - outcome: RelateCompletion::Complete, - claims_related: claims.len(), - supersessions: Vec::new(), - conflicts: Vec::new(), - unresolved: Vec::new(), - held: Vec::new(), - warnings: Vec::new(), - authority_warnings: Vec::new(), - receipts: Vec::new(), - }); - } - let (root, cluster, prefilter, gateway) = env::with(|op_env| { - let cluster = op_env.config.semantics.as_ref().map_or_else( - || crate::config::SemanticsConfig::default().cosine_threshold, - |semantics| semantics.cosine_threshold, - ); - let prefilter = op_env - .config - .semantics - .as_ref() - .and_then(|semantics| semantics.relate_prefilter) - .unwrap_or(RELATE_PREFILTER); - ( - op_env.root.clone(), - cluster, - prefilter, - op_env.config.gateway.clone(), - ) - })?; - let authority = authoritative_settlements(None)?; - let temporal = semantic_temporal_policy(&view)?; - let budget_secs = std::env::var("TEXO_RELATE_BUDGET_SECS") - .ok() - .and_then(|value| value.parse::().ok()) - .or_else(|| gateway.as_ref().map(|config| config.relate_budget_secs)) - .unwrap_or(900); - let mut related = relate_with_backends( - &root, - gateway.as_ref(), - &claims, - RelateThresholds { cluster, prefilter }, - &authority.verdicts, - &temporal, - std::time::Duration::from_secs(budget_secs), - )?; - for (pair, cache_key) in &authority.cache_keys { - related.cache_keys.insert(pair.clone(), cache_key.clone()); - } - let workspace_id = WorkspaceId::try_from(view.workspace_id.as_str())?; - for judgment in &related.outcome.related.judgments { - if judgment.reused_authority { - continue; - } - let cache_key_hex = related - .cache_keys - .get(&( - judgment.older_claim.to_string(), - judgment.newer_claim.to_string(), - )) - .cloned() - .unwrap_or_default(); - append_json( - op, - cx, - ::KIND, - &RelationJudgedV1 { - workspace_id: workspace_id.clone(), - older_claim: judgment.older_claim.clone(), - newer_claim: judgment.newer_claim.clone(), - relation: judgment.verdict.relation.into(), - score_ppm: score_to_ppm(judgment.verdict.score), - judge_fingerprint: related.judge_fingerprint.clone(), - cache_key_hex, - observed_at_ms, - }, - )?; - } - for unresolved in &related.outcome.unresolved { - append_json( - op, - cx, - ::KIND, - &RelationDeferredV1 { - workspace_id: workspace_id.clone(), - older_claim: unresolved.old_claim.clone(), - newer_claim: unresolved.new_claim.clone(), - failure_class: unresolved.failure.class, - attempts: unresolved.failure.attempts, - observed_at_ms, - }, - )?; - } - let partial = !related.outcome.unresolved.is_empty(); - let allow_derived = !strict || !partial; - let mut held = related.outcome.held.clone(); - if strict && partial { - held.extend(related.outcome.related.supersessions.iter().map( - |(old_claim, new_claim, reason)| { - crate::relate::settlement::HeldDecision::Supersession { - old_claim: old_claim.clone(), - new_claim: new_claim.clone(), - reason: reason.clone(), - } - }, - )); - held.extend(related.outcome.related.conflicts.iter().map(|conflict| { - crate::relate::settlement::HeldDecision::Conflict { - conflict_id: conflict.conflict_id.clone(), - claim_a: conflict.claim_a.clone(), - claim_b: conflict.claim_b.clone(), - reason: conflict.reason.clone(), - } - })); - } - let existing_conflicts = view - .conflicts - .iter() - .map(|conflict| conflict.conflict_id.clone()) - .collect::>(); - let mut supersessions = Vec::new(); - let supersession_decisions: &[_] = if allow_derived { - &related.outcome.related.supersessions - } else { - &[] - }; - for (old, new, reason) in supersession_decisions { - let old_id = old.to_string(); - let new_id = new.to_string(); - let old_entity = entity_for_claim(&old_id); - let cache_key = related - .cache_keys - .get(&(old_id.clone(), new_id.clone())) - .cloned() - .unwrap_or_default(); - append_json( - op, - cx, - ::KIND, - &ClaimSupersededV2 { - old_claim_id: old_id.clone(), - new_claim_id: new_id.clone(), - workspace_id: view.workspace_id.clone(), - reason: reason.clone(), - decided_by: "texo-relate".to_string(), - observed_at_ms, - transition: transition_record( - CLAIM_MACHINE, - &old_entity, - 1, - 2, - vec![TransitionCauseV1 { - lane: 0, - key: format!("relate:{cache_key}"), - }], - observed_at_ms, - ), - }, - )?; - supersessions.push(RelateSupersessionRow { - old_claim_id: old_id, - new_claim_id: new_id, - reason: reason.clone(), - cache_key, - }); - } - - let mut conflicts = Vec::new(); - let conflict_decisions: &[_] = if allow_derived { - &related.outcome.related.conflicts - } else { - &[] - }; - for conflict in conflict_decisions { - let conflict_id = conflict.conflict_id.to_string(); - if existing_conflicts.contains(&conflict_id) { - continue; - } - let claim_a = conflict.claim_a.to_string(); - let claim_b = conflict.claim_b.to_string(); - let cache_key = related - .cache_keys - .get(&(claim_a.clone(), claim_b.clone())) - .cloned() - .unwrap_or_default(); - append_json( - op, - cx, - ::KIND, - &ConflictOpenedV2 { - conflict_id: conflict_id.clone(), - workspace_id: view.workspace_id.clone(), - claim_a: claim_a.clone(), - claim_b: claim_b.clone(), - reason: conflict.reason.clone(), - detector: "texo-relate".to_string(), - observed_at_ms, - transition: transition_record( - CONFLICT_MACHINE, - &entity_for_conflict(&conflict_id), - 0, - 1, - vec![TransitionCauseV1 { - lane: 0, - key: format!("relate:{cache_key}"), - }], - observed_at_ms, - ), - }, - )?; - conflicts.push(RelateConflictRow { - conflict_id, - claim_a, - claim_b, - reason: conflict.reason.clone(), - cache_key, - }); - } - - Ok(RelateRunOutput { - outcome: if partial { - RelateCompletion::Partial - } else { - RelateCompletion::Complete - }, - claims_related: claims.len(), - supersessions, - conflicts, - unresolved: related.outcome.unresolved, - held, - warnings: partial - .then(|| { - "semantic settlement is incomplete; unresolved pairs remain authoritative gaps" - .to_string() - }) - .into_iter() - .collect(), - authority_warnings: authority.warnings, - receipts: take_receipts()?, - }) -} - -#[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "score is clamped to the closed 0..=1 interval before ppm conversion" -)] -fn score_to_ppm(score: f32) -> u32 { - (score.clamp(0.0, 1.0) * 1_000_000.0).round() as u32 -} - -struct SemanticRelateOutput { - outcome: crate::semantics::pipeline::RelateOutcome, - cache_keys: BTreeMap<(String, String), String>, - judge_fingerprint: String, -} - -#[cfg(feature = "openrouter")] -fn relate_with_backends( - root: &Path, - gateway: Option<&crate::gateway::GatewayConfig>, - claims: &[(ClaimId, SemanticClaimView)], - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - temporal: &RelateTemporalPolicy, - budget: std::time::Duration, -) -> Result { - use crate::extract::cache::CachingRelater; - use crate::semantics::openrouter::{OpenRouterEmbedder, OpenRouterRelater}; - use crate::semantics::pipeline::relate_claims_settled_parallel_temporal; - use crate::semantics::ClaimRelater as _; - - let embedder = OpenRouterEmbedder::new(None, gateway).map_err(semantic_error)?; - let cache_dir = std::env::var_os(ENV_RELATE_CACHE) - .map_or_else(|| root.join(DEFAULT_RELATE_CACHE), PathBuf::from); - let caching_relater = CachingRelater::new( - OpenRouterRelater::new(None, gateway).map_err(semantic_error)?, - cache_dir, - ); - let judge_fingerprint = caching_relater.fingerprint(); - // Judge calls are independent network waits; fan out across workers and - // reassemble in pair order so settlement stays byte-identical. 4 default - // workers keeps provider pressure polite; clamp guards misconfiguration. - let concurrency = std::env::var("TEXO_RELATE_CONCURRENCY") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(4) - .clamp(1, 16); - let relation_output = relate_claims_settled_parallel_temporal( - claims, - &embedder, - &caching_relater, - thresholds, - settled, - ParallelRelateOptions { - temporal, - budget, - concurrency, - }, - ) - .map_err(semantic_error)?; - let cache_keys = relate_cache_keys(&caching_relater, claims, &relation_output); - Ok(SemanticRelateOutput { - outcome: relation_output, - cache_keys, - judge_fingerprint, - }) -} - -#[cfg(not(feature = "openrouter"))] -fn relate_with_backends( - _root: &Path, - _gateway: Option<&crate::gateway::GatewayConfig>, - _claims: &[(ClaimId, SemanticClaimView)], - _thresholds: RelateThresholds, - _settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - _temporal: &RelateTemporalPolicy, - _budget: std::time::Duration, -) -> Result { - Err(TexoError::Semantics { - backend: "openrouter".to_string(), - detail: "openrouter feature is disabled".to_string(), - }) -} - -#[cfg(feature = "openrouter")] -fn semantic_error(error: impl std::error::Error + Send + Sync + 'static) -> TexoError { - TexoError::Semantics { - backend: "openrouter".to_string(), - detail: crate::error::error_chain(&error), - } -} - -fn semantic_claims_from_view( - view: &WorkspaceView, -) -> Result, TexoError> { - let receipts = claim_record_receipts()?; - let mut claims = Vec::new(); - for claim in &view.claims { - if claim.status != crate::claims::status::ClaimStatus::Current { - continue; - } - let claim_id = ClaimId::try_from(claim.card.claim_id.as_str())?; - let source_id = SourceId::try_from(claim.card.source_id.as_str())?; - let receipt = - receipts - .get(&claim.card.claim_id) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity_for_claim(&claim.card.claim_id), - })?; - let supersedes = claim - .supersedes - .iter() - .map(|id| ClaimId::try_from(id.as_str())) - .collect::, _>>()?; - let superseded_by = claim - .card - .superseded_by - .as_deref() - .map(ClaimId::try_from) - .transpose()?; - claims.push(( - claim_id.clone(), - SemanticClaimView { - claim_id, - workspace_id: claim.card.workspace_id.clone(), - source_id, - source_path: claim.card.source_path.clone(), - line_start: claim.card.line_start, - line_end: claim.card.line_end, - text: claim.card.text.clone(), - normalized_text: claim.card.normalized_text.clone(), - subject_hint: claim.card.subject_hint.clone().unwrap_or_default(), - predicate_hint: claim.card.predicate_hint.clone().unwrap_or_default(), - object_hint: claim.card.object_hint.clone().unwrap_or_default(), - confidence_ppm: claim.card.confidence_ppm, - extractor_kind: claim.card.extractor_kind.clone(), - status: SemanticClaimStatus::Current, - receipt: receipt_view( - 0, - receipt.sequence, - "ClaimRecorded", - &scope_for_workspace(&view.workspace_id), - &entity_for_claim(&claim.card.claim_id), - ), - supersedes, - superseded_by, - }, - )); - } - claims.sort_by(|left, right| { - left.1 - .receipt - .sequence - .get() - .cmp(&right.1.receipt.sequence.get()) - .then_with(|| left.0.as_str().cmp(right.0.as_str())) - }); - Ok(claims) -} - -fn claim_record_receipts() -> Result, TexoError> { - env::with(|op_env| { - let scope = scope_for_workspace(&op_env.workspace_id); - let mut receipts = BTreeMap::new(); - for entry in op_env.store.by_scope(&scope) { - if entry.event_kind() != ::KIND { - continue; - } - // The entity coordinate is "claim:{claim_id}" — the payload decode - // it used to do here (read_raw + msgpack per event) carried no - // information the index entry does not already hold. - let Some(claim_id) = entry.coord().entity().strip_prefix("claim:") else { - continue; - }; - receipts - .entry(claim_id.to_string()) - .or_insert(AgentReceiptRow { - event_id: event_id_hex(entry.event_id()), - sequence: entry.global_sequence(), - }); - } - Ok::<_, TexoError>(receipts) - })? -} - -#[cfg(feature = "openrouter")] -fn relate_cache_keys( - caching_relater: &crate::extract::cache::CachingRelater, - claims: &[(ClaimId, SemanticClaimView)], - relation_output: &crate::semantics::pipeline::RelateOutcome, -) -> BTreeMap<(String, String), String> { - let by_id = claims - .iter() - .map(|(id, view)| (id.to_string(), view)) - .collect::>(); - let mut keys = BTreeMap::new(); - for judgment in &relation_output.related.judgments { - if let (Some(old_view), Some(new_view)) = ( - by_id.get(judgment.older_claim.as_str()), - by_id.get(judgment.newer_claim.as_str()), - ) { - keys.insert( - ( - judgment.older_claim.to_string(), - judgment.newer_claim.to_string(), - ), - caching_relater.cache_key(&old_view.text, &new_view.text), - ); - } - } - keys -} - -fn extract_heuristic_claims( - doc: &MarkdownDocument, - workspace_id: &str, - observed_at_ms: u64, -) -> Result, TexoError> { - let source_id = SourceId::try_from(doc.source_id.as_str())?; - let mut claims = Vec::new(); - for line in &doc.lines { - let normalized = normalize_line(&line.text); - let Some(hints) = hints_from_line_normalized(&line.text, &normalized) else { - continue; - }; - let claim_id = claim_id_from_parts(&source_id, line.number, &normalized).to_string(); - let char_start = saturating_u32(line.char_start); - let char_end = saturating_u32(line.char_start.saturating_add(line.text.len())); - claims.push(ClaimRecordedV2 { - claim_id, - workspace_id: workspace_id.to_string(), - source_id: doc.source_id.clone(), - source_path: doc.path.clone(), - line_start: line.number, - line_end: line.number, - char_start, - char_end, - text: line.text.clone(), - normalized_text: normalized, - subject_hint: Some(hints.subject_hint), - predicate_hint: Some(hints.predicate_hint), - object_hint: Some(hints.object_hint), - confidence_ppm: hints.confidence_ppm, - extractor_kind: "heuristic-v1".to_string(), - extractor_model: String::new(), - prompt_version: String::new(), - observed_at_ms, - }); - } - Ok(claims) -} - -/// Execute the workspace-configured extractor command. -/// -/// This is an explicit local-code-execution trust boundary: anyone who can -/// write workspace configuration selects code executed by the extractor. Texo -/// stages only the selected input and runs the command through the fail-closed -/// bvisor adapter; there is no unconfined fallback. -fn extract_cmd_claims( - op: &str, - _root: &Path, - cmd: &str, - path: &Path, - doc: &MarkdownDocument, - workspace_id: &str, - observed_at_ms: u64, -) -> Result, TexoError> { - let output = - crate::compat::bvisor::run_extractor(cmd, path).map_err(|error| TexoError::Extract { - detail: format!("{op}: confined extractor failed: {error}"), - })?; - let stdout = String::from_utf8(output).map_err(|error| TexoError::Extract { - detail: format!("{op}: extractor stdout was not utf-8: {error}"), - })?; - let source_id = SourceId::try_from(doc.source_id.as_str())?; - let mut claims = Vec::new(); - for (idx, line) in stdout.lines().enumerate() { - if line.trim().is_empty() { - continue; - } - let parsed: CmdClaimLine = - serde_json::from_str(line).map_err(|error| TexoError::Extract { - detail: format!("{op}: extractor line {} json error: {error}", idx + 1), - })?; - let claim_id = - claim_id_from_parts(&source_id, parsed.line_start, &parsed.normalized_text).to_string(); - claims.push(ClaimRecordedV2 { - claim_id, - workspace_id: workspace_id.to_string(), - source_id: doc.source_id.clone(), - source_path: doc.path.clone(), - line_start: parsed.line_start, - line_end: parsed.line_start, - char_start: parsed.char_start.unwrap_or(0), - char_end: parsed.char_end.unwrap_or(0), - text: parsed.text, - normalized_text: parsed.normalized_text, - subject_hint: parsed.subject_hint, - predicate_hint: parsed.predicate_hint, - object_hint: parsed.object_hint, - confidence_ppm: parsed.confidence_ppm, - extractor_kind: "extractor-cmd".to_string(), - extractor_model: parsed.extractor_model.unwrap_or_default(), - prompt_version: parsed.prompt_version.unwrap_or_default(), - observed_at_ms, - }); - } - Ok(claims) -} - -fn saturating_u32(value: usize) -> u32 { - // Source byte offsets are journaled as v2 u32 fields; extremely large inputs - // saturate rather than truncating silently. - u32::try_from(value).unwrap_or(u32::MAX) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -/// Closed reason an explicit replacement proposal could not become authority. -pub enum ExplicitSupersessionHoldReason { - /// The proposed successor is on an older source revision. - TemporalReversed, - /// Both source revisions are valid but incomparable. - TemporalConcurrent, - /// Available source evidence cannot establish an order. - TemporalUnknown, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -/// One explicit replacement proposal held by temporal policy. -pub struct HeldExplicitSupersession { - /// Claim that would have been retired. - pub old_claim_id: ClaimId, - /// Proposed successor claim. - pub new_claim_id: ClaimId, - /// Typed reason no authority-bearing event was appended. - pub reason: ExplicitSupersessionHoldReason, -} - -/// Applied and held outcomes from the shared explicit-replacement policy. -pub(crate) struct ExplicitSupersessionOutcome { - /// Authority-bearing supersession events safe to append. - pub applied: Vec, - /// Proposals withheld pending authoritative source order. - pub held: Vec, -} - -#[derive(Clone)] -struct ExplicitReplacementCandidate { - claim_id: ClaimId, - workspace_id: String, - source_id: String, - normalized_text: String, - subject_hint: Option, -} - -pub(crate) fn infer_supersessions( - view: &WorkspaceView, - new_claims: &[ClaimRecordedV2], - observed_at_ms: u64, - temporal: &RelateTemporalPolicy, -) -> Result { - let candidates = new_claims - .iter() - .map(|claim| { - Ok(ExplicitReplacementCandidate { - claim_id: ClaimId::try_from(claim.claim_id.as_str())?, - workspace_id: claim.workspace_id.clone(), - source_id: claim.source_id.clone(), - normalized_text: claim.normalized_text.clone(), - subject_hint: claim.subject_hint.clone(), - }) - }) - .collect::, TexoError>>()?; - Ok(infer_explicit_supersessions( - view, - &candidates, - observed_at_ms, - temporal, - )) -} - -fn infer_indexed_supersessions( - view: &WorkspaceView, - new_claim_ids: &BTreeSet, - observed_at_ms: u64, - temporal: &RelateTemporalPolicy, -) -> ExplicitSupersessionOutcome { - let candidates = view - .claims - .iter() - .filter_map(|claim| { - let claim_id = ClaimId::try_from(claim.card.claim_id.as_str()).ok()?; - new_claim_ids - .contains(&claim_id) - .then(|| ExplicitReplacementCandidate { - claim_id, - workspace_id: claim.card.workspace_id.clone(), - source_id: claim.card.source_id.clone(), - normalized_text: claim.card.normalized_text.clone(), - subject_hint: claim.card.subject_hint.clone(), - }) - }) - .collect::>(); - infer_explicit_supersessions(view, &candidates, observed_at_ms, temporal) -} - -fn settle_indexed_explicit_supersessions( - cx: &mut syncbat::Ctx<'_>, - view: &WorkspaceView, - new_claim_ids: &BTreeSet, - observed_at_ms: u64, - receipts: &mut Vec, -) -> Result { - let evidence_frontier = receipts - .iter() - .map(|receipt| receipt.global_sequence) - .max() - .unwrap_or(view.frontier); - let temporal = workspace_temporal_policy_through(view, evidence_frontier)?; - let outcome = infer_indexed_supersessions(view, new_claim_ids, observed_at_ms, &temporal); - for supersession in &outcome.applied { - append_json( - "texo.knowledge.index", - cx, - ::KIND, - supersession, - )?; - } - receipts.extend(take_receipts()?); - Ok(outcome) -} - -fn infer_explicit_supersessions( - view: &WorkspaceView, - new_claims: &[ExplicitReplacementCandidate], - observed_at_ms: u64, - temporal: &RelateTemporalPolicy, -) -> ExplicitSupersessionOutcome { - let mut by_subject: BTreeMap, Vec<&ClaimView>> = BTreeMap::new(); - for claim in &view.claims { - by_subject - .entry(claim.card.subject_hint.clone()) - .or_default() - .push(claim); - } - let mut applied = Vec::new(); - let mut held = Vec::new(); - let mut seen_old = BTreeSet::new(); - for new_claim in new_claims { - if !replacement_signal(&new_claim.normalized_text) { - continue; - } - let Some(candidates) = by_subject.get(&new_claim.subject_hint) else { - continue; - }; - for old in candidates { - if old.card.claim_id == new_claim.claim_id.as_str() - || old.card.phase != 1 - || old.card.normalized_text == new_claim.normalized_text - { - continue; - } - let Ok(old_claim_id) = ClaimId::try_from(old.card.claim_id.as_str()) else { - continue; - }; - let hold_reason = match temporal.compare_claims(&old_claim_id, &new_claim.claim_id) { - None | Some(TemporalRelation::Same | TemporalRelation::Before) => None, - Some(TemporalRelation::After) => { - Some(ExplicitSupersessionHoldReason::TemporalReversed) - } - Some(TemporalRelation::Concurrent) => { - Some(ExplicitSupersessionHoldReason::TemporalConcurrent) - } - Some(TemporalRelation::Unknown) => { - Some(ExplicitSupersessionHoldReason::TemporalUnknown) - } - }; - if !seen_old.insert(old.card.claim_id.clone()) { - continue; - } - if let Some(reason) = hold_reason { - held.push(HeldExplicitSupersession { - old_claim_id, - new_claim_id: new_claim.claim_id.clone(), - reason, - }); - continue; - } - let old_entity = entity_for_claim(&old.card.claim_id); - applied.push(ClaimSupersededV2 { - old_claim_id: old.card.claim_id.clone(), - new_claim_id: new_claim.claim_id.to_string(), - workspace_id: new_claim.workspace_id.clone(), - reason: "explicit replacement wording accepted by temporal policy".to_string(), - decided_by: "texo.ingest.run".to_string(), - observed_at_ms, - transition: transition_record( - CLAIM_MACHINE, - &old_entity, - 1, - 2, - vec![TransitionCauseV1 { - lane: 0, - key: format!("ingest:{}", new_claim.source_id), - }], - observed_at_ms, - ), - }); - } - } - applied.sort_by(|left, right| { - left.old_claim_id - .cmp(&right.old_claim_id) - .then_with(|| left.new_claim_id.cmp(&right.new_claim_id)) - }); - held.sort_by(|left, right| { - left.old_claim_id - .cmp(&right.old_claim_id) - .then_with(|| left.new_claim_id.cmp(&right.new_claim_id)) - }); - ExplicitSupersessionOutcome { applied, held } -} - -fn replacement_signal(normalized_text: &str) -> bool { - crate::lexicon::contains_replacement_signal(normalized_text) -} - -fn claim_list_rows( - view: &WorkspaceView, - subject: Option<&str>, -) -> Result, TexoError> { - let mut rows = Vec::new(); - // One index scan for all receipts instead of one by_entity per claim. - let receipts = claim_record_receipts()?; - for claim in &view.claims { - if subject.is_some_and(|wanted| claim.card.subject_hint.as_deref() != Some(wanted)) { - continue; - } - let receipt = match receipts.get(&claim.card.claim_id) { - Some(row) => row.clone(), - None => claim_receipt(&claim.card.claim_id)?, - }; - rows.push(AgentClaimRow { - claim_id: claim.card.claim_id.clone(), - status: claim.status, - subject_hint: claim.card.subject_hint.clone(), - text: claim.card.text.clone(), - source: AgentSourceRow { - source_id: claim.card.source_id.clone(), - path: claim.card.source_path.clone(), - line_start: claim.card.line_start, - }, - receipt, - supersedes: claim.supersedes.clone(), - superseded_by: claim.card.superseded_by.clone(), - }); - } - Ok(rows) -} - -fn search_knowledge_from_view( - view: &WorkspaceView, - snapshot: SnapshotRead, - input: &KnowledgeSearchInput, -) -> Result { - let query = input.query.as_deref().unwrap_or(""); - if query.len() > 256 { - return Err(TexoError::OpInput { - op: "texo.knowledge.search".to_string(), - detail: "query exceeds 256 bytes".to_string(), - }); - } - let limit = input.limit.unwrap_or(25); - if !(1..=100).contains(&limit) { - return Err(TexoError::OpInput { - op: "texo.knowledge.search".to_string(), - detail: "limit must be between 1 and 100".to_string(), - }); - } - let cursor_identity = knowledge_search_identity(&snapshot, query, input); - let offset = parse_knowledge_search_cursor(input.cursor.as_deref(), &cursor_identity)?; - let query_lower = query.to_ascii_lowercase(); - let mut ranked = claim_list_rows(view, input.subject.as_deref())? - .into_iter() - .filter(|claim| input.status.is_none_or(|status| claim.status == status)) - .filter_map(|claim| { - claim_search_rank(&claim, &query_lower).map(|rank| RankedKnowledgeResult { - rank, - key: format!("claim:{}", claim.claim_id), - result: KnowledgeSearchResult::Claim { claim }, - }) - }) - .collect::>(); - let mut coverage = coverage_for_view(view, &snapshot)?; - // A subject/status filter searches only the claim plane; the code plane is - // deliberately not consulted, which is "not searched", not "unavailable". - let code_searched = input.subject.is_none() && input.status.is_none(); - let mut code_index_available = false; - if code_searched { - if let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() { - let loaded = load_code_artifact_at(view.frontier, source_snapshot_id)?; - if let Some(code_coverage) = &loaded.coverage { - merge_coverage(&mut coverage, code_coverage); - } - if let Some(artifact) = loaded.artifact { - code_index_available = true; - ranked.extend(artifact.occurrences.into_iter().filter_map(|occurrence| { - code_search_rank(&occurrence, &query_lower).map(|rank| RankedKnowledgeResult { - rank, - key: format!( - "code:{}:{}:{}", - occurrence.symbol, occurrence.path, occurrence.byte_range.start - ), - result: KnowledgeSearchResult::Code { occurrence }, - }) - })); - } - } - } - // Only assert the code index is unavailable when it was actually searched and - // found absent — never for a filtered search that skipped it by design. - if code_searched - && !code_index_available - && !coverage - .gaps - .iter() - .any(|gap| gap.kind == CoverageGapKind::CodeIndexUnavailable) - { - coverage.gaps.push(CoverageGap { - path: None, - kind: CoverageGapKind::CodeIndexUnavailable, - }); - } - ranked.sort_by(|left, right| { - left.rank - .cmp(&right.rank) - .then_with(|| left.key.cmp(&right.key)) - }); - let total = ranked.len(); - let results = ranked - .into_iter() - .skip(offset) - .take(limit) - .map(|ranked| ranked.result) - .collect::>(); - let returned = results.len(); - let next_offset = offset.saturating_add(returned); - let has_more = next_offset < total; - Ok(KnowledgeSearchOutput { - workspace_id: view.workspace_id.clone(), - frontier: view.frontier, - total, - returned, - has_more, - next_cursor: has_more.then(|| format!("texo-knowledge-v1:{cursor_identity}:{next_offset}")), - results, - code_index_available, - coverage, - snapshot, - }) -} - -fn knowledge_search_identity( - snapshot: &SnapshotRead, - query: &str, - input: &KnowledgeSearchInput, -) -> String { - let material = format!( - "texo.knowledge.search.v1\u{1f}{}\u{1f}{query}\u{1f}{}\u{1f}{}", - snapshot.token.as_str(), - input.subject.as_deref().unwrap_or(""), - input - .status - .map_or("", crate::claims::status::ClaimStatus::as_str) - ); - blake3::hash(material.as_bytes()).to_hex()[..16].to_string() -} - -fn parse_knowledge_search_cursor( - cursor: Option<&str>, - expected_identity: &str, -) -> Result { - let Some(cursor) = cursor else { - return Ok(0); - }; - let Some(rest) = cursor.strip_prefix("texo-knowledge-v1:") else { - return Err(invalid_knowledge_cursor()); - }; - let Some((identity, offset)) = rest.split_once(':') else { - return Err(invalid_knowledge_cursor()); - }; - if identity != expected_identity { - return Err(invalid_knowledge_cursor()); - } - offset.parse().map_err(|_| invalid_knowledge_cursor()) -} - -fn invalid_knowledge_cursor() -> TexoError { - TexoError::OpInput { - op: "texo.knowledge.search".to_string(), - detail: "cursor is invalid for this query and snapshot".to_string(), - } -} - -fn claim_search_rank(claim: &AgentClaimRow, query: &str) -> Option { - if query.is_empty() { - return Some(0); - } - let text = claim.text.to_ascii_lowercase(); - let subject = claim - .subject_hint - .as_deref() - .unwrap_or("") - .to_ascii_lowercase(); - let path = claim.source.path.to_ascii_lowercase(); - if text == query || subject == query { - Some(0) - } else if text.starts_with(query) || subject.starts_with(query) { - Some(1) - } else if text.contains(query) || subject.contains(query) { - Some(2) - } else if path.contains(query) { - Some(3) - } else { - None - } -} - -fn code_search_rank(occurrence: &CodeOccurrence, query: &str) -> Option { - if query.is_empty() { - return Some(1); - } - let name = occurrence.display_name.to_ascii_lowercase(); - let symbol = occurrence.symbol.to_ascii_lowercase(); - let path = occurrence.path.to_ascii_lowercase(); - if name == query || symbol == query { - Some(0) - } else if name.starts_with(query) { - Some(1) - } else if name.contains(query) || symbol.contains(query) { - Some(2) - } else if path.contains(query) { - Some(3) - } else { - None - } -} - -fn parse_claim_search_cursor(cursor: Option<&str>) -> Result { - let Some(cursor) = cursor else { - return Ok(0); - }; - let Some(offset) = cursor.strip_prefix("texo-claims-v1:") else { - return Err(TexoError::OpInput { - op: "texo.claims.search".to_string(), - detail: "cursor has an unsupported schema".to_string(), - }); - }; - offset.parse::().map_err(|_| TexoError::OpInput { - op: "texo.claims.search".to_string(), - detail: "cursor offset is invalid".to_string(), - }) -} - -fn claim_matches_query(row: &AgentClaimRow, terms: &[String]) -> bool { - if terms.is_empty() { - return true; - } - let mut searchable = row.text.to_ascii_lowercase(); - searchable.push(' '); - searchable.push_str(&row.source.path.to_ascii_lowercase()); - if let Some(subject) = &row.subject_hint { - searchable.push(' '); - searchable.push_str(&subject.to_ascii_lowercase()); - } - terms.iter().all(|term| searchable.contains(term)) -} - -fn build_agent_context_from_view( - view: &WorkspaceView, - subject: Option<&str>, - include_stale: bool, - snapshot: SnapshotRead, -) -> Result { - let claims = claim_list_rows(view, subject)? - .into_iter() - .filter(|claim| claim.status != crate::claims::status::ClaimStatus::Superseded) - .collect::>(); - let stale_claims = if include_stale { - view.claims - .iter() - .filter(|claim| claim.card.phase == 2) - .filter(|claim| { - subject.is_none_or(|wanted| claim.card.subject_hint.as_deref() == Some(wanted)) - }) - .filter_map(|claim| { - claim - .card - .superseded_by - .as_ref() - .map(|superseded_by| AgentStaleClaimRow { - claim_id: claim.card.claim_id.clone(), - text: claim.card.text.clone(), - superseded_by: superseded_by.clone(), - }) - }) - .collect() - } else { - Vec::new() - }; - - let mut conflicts = Vec::new(); - for conflict in &view.conflicts { - if conflict.phase != 1 { - continue; - } - conflicts.push(AgentConflictRow { - conflict_id: conflict.conflict_id.clone(), - claim_a: conflict.claim_a.clone(), - claim_a_text: claim_text(view, &conflict.claim_a), - claim_b: conflict.claim_b.clone(), - claim_b_text: claim_text(view, &conflict.claim_b), - reason: conflict.reason.clone(), - }); - } - conflicts.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); - let mut seen_pairs = BTreeSet::new(); - conflicts.retain(|conflict| { - let mut pair = [ - conflict.claim_a_text.to_ascii_lowercase(), - conflict.claim_b_text.to_ascii_lowercase(), - ]; - pair.sort(); - seen_pairs.insert(pair) - }); - - Ok(AgentContextOutput { - workspace_id: view.workspace_id.clone(), - replayed_through_sequence: view.frontier, - freshness: FreshnessView { - kind: view.freshness, - description: format!( - "Projection anchor validated through local store sequence {}. No global order or consensus is claimed.", - view.frontier - ), - }, - claims, - stale_claims, - conflicts, - snapshot, - }) -} - -fn check_staleness_from_view( - view: &WorkspaceView, - workspace_id: &str, - root: &Path, - input: &Path, - snapshot: SnapshotRead, -) -> Result { - let checked_path = input - .strip_prefix(root) - .unwrap_or(input) - .to_string_lossy() - .to_string(); - let discovery = collect_markdown_files(input).map_err(|error| TexoError::Source { - path: input.to_string_lossy().to_string(), - detail: error.to_string(), - })?; - if let Some(failure) = discovery.failures.first() { - return Err(TexoError::Source { - path: failure.path.to_string_lossy().to_string(), - detail: failure.error.to_string(), - }); - } - let by_id = view - .claims - .iter() - .map(|claim| (claim.card.claim_id.clone(), claim)) - .collect::>(); - let mut diagnostics = Vec::new(); - for path in discovery.files { - let doc = MarkdownDocument::from_path(&path, root).map_err(|error| TexoError::Source { - path: path.to_string_lossy().to_string(), - detail: error.to_string(), - })?; - let source_id = SourceId::try_from(doc.source_id.as_str())?; - // Match superseded claims of THIS doc by normalized-text containment in - // the doc's current lines. Reconstructing claim ids from whole lines - // only matches heuristic whole-line claims; LLM extraction proposes - // sub-sentence claims whose identity a line-level rebuild never hits. - // Normalize each doc line once, not once per superseded claim. - let normalized_lines = doc - .lines - .iter() - .map(|line| (line, normalize_line(&line.text))) - .collect::>(); - for claim in &view.claims { - if claim.card.phase != 2 || claim.card.source_id != source_id.as_str() { - continue; - } - let needle = claim.card.normalized_text.as_str(); - if needle.is_empty() { - continue; - } - let line = normalized_lines - .iter() - .find(|(line, normalized)| { - line.number == claim.card.line_start && normalized.contains(needle) - }) - .or_else(|| { - normalized_lines - .iter() - .find(|(_, normalized)| normalized.contains(needle)) - }) - .map(|(line, _)| *line); - let Some(line) = line else { - continue; // the stale text no longer appears in the doc - }; - let superseded_by = claim.card.superseded_by.clone(); - let source = superseded_by - .as_ref() - .and_then(|id| by_id.get(id)) - .map(|superseder| DiagnosticSource { - path: superseder.card.source_path.clone(), - line_start: superseder.card.line_start, - }); - let receipt = superseded_by - .as_ref() - .and_then(|id| claim_receipt(id).ok()) - .or_else(|| claim_receipt(&claim.card.claim_id).ok()); - let message = format!( - "Claim appears stale: superseded by {} at {}.", - superseded_by.as_deref().unwrap_or("unknown"), - receipt.as_ref().map_or_else( - || "unknown seq".to_string(), - |receipt| format!("local seq {}", receipt.sequence) - ) - ); - diagnostics.push(StaleDiagnostic { - file: doc.path.clone(), - line_start: line.number, - line_end: line.number, - severity: DiagnosticSeverity::Warning, - message, - claim_id: claim.card.claim_id.clone(), - superseded_by, - source, - receipt, - }); - } - } - Ok(StalenessReport { - workspace_id: workspace_id.to_string(), - checked_path, - replayed_through_sequence: view.frontier, - diagnostics, - snapshot, - }) -} - -fn compile_artifacts( - context: &AgentContextOutput, - view: &WorkspaceView, - stale: &StalenessReport, - conflicts: &heuristic::ConflictReport, -) -> Result, TexoError> { - Ok(vec![ - CompileFile { - name: "onboarding.generated.md".to_string(), - contents: render_onboarding(context), - }, - CompileFile { - name: "claims.json".to_string(), - contents: serde_json::to_string_pretty(view)?, - }, - CompileFile { - name: "stale-context.json".to_string(), - contents: serde_json::to_string_pretty(stale)?, - }, - CompileFile { - name: "conflicts.json".to_string(), - contents: serde_json::to_string_pretty(conflicts)?, - }, - CompileFile { - name: "agent-context.json".to_string(), - contents: serde_json::to_string_pretty(context)?, - }, - CompileFile { - name: "index.html".to_string(), - contents: render_index_html(context, stale, conflicts)?, - }, - ]) -} - -fn render_onboarding(context: &AgentContextOutput) -> String { - let mut out = String::from("# Generated Onboarding\n\n"); - out.push_str( - "_This document is a projection replayed from the texo claim-chain. \ - It is not source truth._\n\n", - ); - writeln!( - &mut out, - "_Replayed through local store sequence {}._\n", - context.replayed_through_sequence - ) - .expect("writing to a String cannot fail"); - out.push_str("## Current claims\n\n"); - for claim in &context.claims { - writeln!( - &mut out, - "- **{}** ({}): {} \n _source: {}:{}_", - claim.claim_id, - claim.subject_hint.clone().unwrap_or_default(), - claim.text, - claim.source.path, - claim.source.line_start - ) - .expect("writing to a String cannot fail"); - } - if !context.stale_claims.is_empty() { - out.push_str("\n## Stale claims (do not trust)\n\n"); - for stale in &context.stale_claims { - writeln!( - &mut out, - "- {}: \"{}\" superseded by {}", - stale.claim_id, stale.text, stale.superseded_by - ) - .expect("writing to a String cannot fail"); - } - } - if !context.conflicts.is_empty() { - out.push_str("\n## Conflicts (unresolved — both claimed, neither wins)\n\n"); - for conflict in &context.conflicts { - writeln!( - &mut out, - "- \"{}\" ({}) vs \"{}\" ({})", - conflict.claim_a_text, conflict.claim_a, conflict.claim_b_text, conflict.claim_b - ) - .expect("writing to a String cannot fail"); - } - } - out -} - -fn render_index_html( - context: &AgentContextOutput, - stale: &StalenessReport, - conflicts: &heuristic::ConflictReport, -) -> Result { - let mut claim_cards = String::new(); - for claim in &context.claims { - let supersedes = if claim.supersedes.is_empty() { - String::new() - } else { - format!( - "

supersedes: {}

", - claim.supersedes.join(", ") - ) - }; - write!( - &mut claim_cards, - r#"
-

Claim {id}

-

status: current

-

subject: {subject}

-

local sequence: {seq}

-

frontier: replayed through seq {frontier}

-

source: {path}:{line}

-

receipt: {receipt}

- {supersedes} -
{text}
-
"#, - id = claim.claim_id, - subject = claim.subject_hint.clone().unwrap_or_default(), - seq = claim.receipt.sequence, - frontier = context.replayed_through_sequence, - path = claim.source.path, - line = claim.source.line_start, - receipt = claim.receipt.event_id, - supersedes = supersedes, - text = html_escape(&claim.text), - ) - .expect("writing to a String cannot fail"); - } - let mut stale_cards = String::new(); - for diag in &stale.diagnostics { - write!( - &mut stale_cards, - r#"
-

Stale line {}:{}

-

{}

-
"#, - diag.file, - diag.line_start, - html_escape(&diag.message) - ) - .expect("writing to a String cannot fail"); - } - let conflicts_json = serde_json::to_string_pretty(conflicts)?; - Ok(format!( - r#" - - - - texo claim explorer - - - -
-

A block explorer for stale team beliefs.

-

Every claim below was replayed from a BatPak journal. The generated onboarding doc is a projection, not source truth.

-
-
-

Current claims

- {claim_cards} -
-
-

Stale diagnostics

- {stale_cards} -
-
-

Conflicts ({conflict_count})

- 
{conflicts_json}
-
-
- texo uses one local BatPak journal. Sequences are per-store. No global order, network consensus, or distributed replication is claimed. -
- -"#, - conflict_count = conflicts.conflicts.len(), - conflicts_json = html_escape(&conflicts_json) - )) -} - -fn html_escape(text: &str) -> String { - text.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) -} - -fn conflicts_output(view: &WorkspaceView) -> ConflictsOutput { - let mut open = Vec::new(); - let mut resolved = Vec::new(); - for conflict in &view.conflicts { - let row = ConflictRow { - conflict_id: conflict.conflict_id.clone(), - claim_a: conflict.claim_a.clone(), - claim_b: conflict.claim_b.clone(), - subject_hint: conflict_subject(view, conflict), - reason: conflict.reason.clone(), - status: conflict_status(conflict), - }; - if conflict.phase == 1 { - open.push(row); - } else { - resolved.push(row); - } - } - open.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); - resolved.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); - ConflictsOutput { open, resolved } -} - -fn conflict_status(conflict: &ConflictCard) -> crate::claims::status::ConflictStatus { - match conflict.phase { - 2 => crate::claims::status::ConflictStatus::Resolved, - 3 => crate::claims::status::ConflictStatus::Ignored, - _ => crate::claims::status::ConflictStatus::Open, - } -} - -fn claim_phase_name(phase: u64) -> &'static str { - match phase { - 0 => "unrecorded", - 1 => "current", - 2 => "superseded", - _ => "invalid", - } -} - -fn conflict_phase_name(phase: u64) -> &'static str { - match phase { - 0 => "unopened", - 1 => "open", - 2 => "resolved", - 3 => "ignored", - _ => "invalid", - } -} - -fn workspace_event_count() -> Result { - env::with(|op_env| { - let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); - let mut after = None; - let mut count = 0usize; - loop { - let page = op_env.store.query_entries_after(®ion, after, 256); - if page.is_empty() { - break; - } - count = count.saturating_add(page.len()); - after = page.last().map(batpak::store::IndexEntry::global_sequence); - } - count - }) -} - -fn file_bytes(path: &Path) -> Result { - match std::fs::metadata(path) { - Ok(metadata) => Ok(metadata.len()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0), - Err(error) => Err(error.into()), - } -} - -fn journal_file_bytes(path: &Path) -> Result { - let metadata = match std::fs::metadata(path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(error) => return Err(error.into()), - }; - if metadata.is_file() { - return Ok( - if path.extension().and_then(std::ffi::OsStr::to_str) == Some("fbat") { - metadata.len() - } else { - 0 - }, - ); - } - let mut bytes = 0u64; - for entry in std::fs::read_dir(path)? { - bytes = bytes.saturating_add(journal_file_bytes(&entry?.path())?); - } - Ok(bytes) -} - -fn conflict_subject(view: &WorkspaceView, conflict: &ConflictCard) -> String { - view.claims - .iter() - .find(|claim| claim.card.claim_id == conflict.claim_a) - .and_then(|claim| claim.card.subject_hint.clone()) - .unwrap_or_default() -} - -fn claim_text(view: &WorkspaceView, claim_id: &str) -> String { - view.claims - .iter() - .find(|claim| claim.card.claim_id == claim_id) - .map(|claim| claim.card.text.clone()) - .unwrap_or_default() -} - -fn newer_claim<'a>( - view: &'a WorkspaceView, - left: &str, - right: &str, -) -> Result<&'a ClaimCard, TexoError> { - let left = view - .claims - .iter() - .find(|claim| claim.card.claim_id == left) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity_for_claim(left), - })?; - let right = view - .claims - .iter() - .find(|claim| claim.card.claim_id == right) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity_for_claim(right), - })?; - if left.card.observed_at_ms >= right.card.observed_at_ms { - Ok(&left.card) - } else { - Ok(&right.card) - } -} - -fn claim_receipt(claim_id: &str) -> Result { - let entity = entity_for_claim(claim_id); - env::with(|op_env| { - let entry = op_env - .store - .by_entity(&entity) - .into_iter() - .find(|entry| entry.event_kind() == ::KIND) - .ok_or_else(|| TexoError::MissingEntity { - entity: entity.clone(), - })?; - Ok::<_, TexoError>(AgentReceiptRow { - event_id: event_id_hex(entry.event_id()), - sequence: entry.global_sequence(), - }) - })? -} - -fn validate_transition_edges(view: &WorkspaceView, errors: &mut Vec) -> bool { - let mut ok = true; - for claim in &view.claims { - let edge = match claim.card.phase { - 1 => Some((0, 1)), - 2 => Some((1, 2)), - _ => None, - }; - if edge.is_none_or(|edge| !CLAIM_EDGES.contains(&edge)) { - ok = false; - errors.push(format!( - "claim {} invalid phase {} for {CLAIM_MACHINE}", - claim.card.claim_id, claim.card.phase - )); - } - if claim.card.phase == 2 && claim.card.superseded_by.is_none() { - ok = false; - errors.push(format!( - "claim {} superseded without target", - claim.card.claim_id - )); - } - } - for conflict in &view.conflicts { - let edge = match conflict.phase { - 1 => Some((0, 1)), - 2 => Some((1, 2)), - 3 => Some((1, 3)), - _ => None, - }; - if edge.is_none_or(|edge| !CONFLICT_EDGES.contains(&edge)) { - ok = false; - errors.push(format!( - "conflict {} invalid phase {} for {CONFLICT_MACHINE}", - conflict.conflict_id, conflict.phase - )); - } - } - ok -} - -fn event_id_hex(event_id: batpak::id::EventId) -> String { - format!("{:032x}", event_id.as_u128()) -} diff --git a/src/ops/handlers/agent_context.rs b/src/ops/handlers/agent_context.rs new file mode 100644 index 0000000..16fff85 --- /dev/null +++ b/src/ops/handlers/agent_context.rs @@ -0,0 +1,174 @@ +use super::claims::claim_list_rows; +use super::common::{ + assemble_snapshot_view, op_runtime, parse_input, run_op, WORKSPACE_VIEW_PROJECTION, +}; +use super::conflicts::claim_text; +use super::model::AgentClaimRow; +use super::relate::require_complete_settlement; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::knowledge::SnapshotRead; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = CONTEXT_AGENT, + register = register_context_agent, + register_item = context_agent_item, + name = "texo.context.agent", + effect = Inspect, + input_schema = "texo.context.agent.input.v4", + output_schema = "texo.context.agent.output.v3", + receipt_kind = "receipt.texo.context.agent.v3", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn context_agent(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.context.agent", || { + let input: ContextAgentInput = parse_input("texo.context.agent", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.context.agent", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + if !input.allow_unsettled { + require_complete_settlement(&view)?; + } + build_agent_context_from_view( + &view, + input.subject.as_deref(), + input.include_stale, + snapshot, + ) + }) +} + +#[derive(Debug, Deserialize)] +struct ContextAgentInput { + subject: Option, + include_stale: bool, + #[serde(default)] + allow_unsettled: bool, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct AgentContextOutput { + pub(super) workspace_id: String, + pub(super) replayed_through_sequence: u64, + pub(super) freshness: FreshnessView, + pub(super) claims: Vec, + pub(super) stale_claims: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) conflicts: Vec, + pub(super) snapshot: SnapshotRead, +} + +#[derive(Debug, Serialize)] +pub(super) struct FreshnessView { + pub(super) kind: crate::claims::workspace::ProjectionFreshness, + pub(super) description: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct AgentStaleClaimRow { + pub(super) claim_id: String, + pub(super) text: String, + pub(super) superseded_by: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct AgentConflictRow { + pub(super) conflict_id: String, + pub(super) claim_a: String, + pub(super) claim_a_text: String, + pub(super) claim_b: String, + pub(super) claim_b_text: String, + pub(super) reason: String, +} + +pub(super) fn build_agent_context_from_view( + view: &WorkspaceView, + subject: Option<&str>, + include_stale: bool, + snapshot: SnapshotRead, +) -> Result { + let claims = claim_list_rows(view, subject)? + .into_iter() + .filter(|claim| claim.status != crate::claims::status::ClaimStatus::Superseded) + .collect::>(); + let stale_claims = stale_claim_rows(view, subject, include_stale); + let conflicts = conflict_rows(view); + + Ok(AgentContextOutput { + workspace_id: view.workspace_id.clone(), + replayed_through_sequence: view.frontier, + freshness: FreshnessView { + kind: view.freshness, + description: format!( + "Projection anchor validated through local store sequence {}. No global order or consensus is claimed.", + view.frontier + ), + }, + claims, + stale_claims, + conflicts, + snapshot, + }) +} + +fn stale_claim_rows( + view: &WorkspaceView, + subject: Option<&str>, + include_stale: bool, +) -> Vec { + if !include_stale { + return Vec::new(); + } + view.claims + .iter() + .filter(|claim| claim.card.phase == 2) + .filter(|claim| { + subject.is_none_or(|wanted| claim.card.subject_hint.as_deref() == Some(wanted)) + }) + .filter_map(|claim| { + claim + .card + .superseded_by + .as_ref() + .map(|superseded_by| AgentStaleClaimRow { + claim_id: claim.card.claim_id.clone(), + text: claim.card.text.clone(), + superseded_by: superseded_by.clone(), + }) + }) + .collect() +} + +fn conflict_rows(view: &WorkspaceView) -> Vec { + let mut conflicts = view + .conflicts + .iter() + .filter(|conflict| conflict.phase == 1) + .map(|conflict| AgentConflictRow { + conflict_id: conflict.conflict_id.clone(), + claim_a: conflict.claim_a.clone(), + claim_a_text: claim_text(view, &conflict.claim_a), + claim_b: conflict.claim_b.clone(), + claim_b_text: claim_text(view, &conflict.claim_b), + reason: conflict.reason.clone(), + }) + .collect::>(); + conflicts.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); + let mut seen_pairs = BTreeSet::new(); + conflicts.retain(|conflict| { + let mut pair = [ + conflict.claim_a_text.to_ascii_lowercase(), + conflict.claim_b_text.to_ascii_lowercase(), + ]; + pair.sort(); + seen_pairs.insert(pair) + }); + conflicts +} diff --git a/src/ops/handlers/claims.rs b/src/ops/handlers/claims.rs new file mode 100644 index 0000000..bc6eead --- /dev/null +++ b/src/ops/handlers/claims.rs @@ -0,0 +1,667 @@ +use super::common::{ + append_json, assemble_snapshot_view, claim_receipt, claim_record_receipts, + claim_timeline_through, coverage_for_view, evidence_projection_through, op_runtime, + parse_input, run_op, take_one_receipt, WORKSPACE_VIEW_PROJECTION, +}; +use super::knowledge_read::{answer_state_for_claim, load_code_artifact_at, merge_coverage}; +use super::model::{AgentClaimRow, AgentSourceRow}; +use super::stats::claim_phase_name; +use crate::claims::card::ClaimCard; +use crate::claims::timeline::TimelineEntry; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::coordinate::entity_for_claim; +use crate::events::machines::{transition_record, TransitionCauseV1, CLAIM_MACHINE}; +use crate::events::payloads::ClaimSupersededV2; +use crate::knowledge::{ + AnswerState, ClaimEvidence, CodeOccurrence, CoverageGap, CoverageGapKind, KnowledgeCoverage, + SnapshotRead, +}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use batpak::event::EventPayload; +use batpak::store::Freshness; +use serde::{Deserialize, Serialize}; +use syncbat::HandlerResult; + +const CLAIM_EXPLAIN_PROJECTION: &str = "texo.claim.explain.v2"; + +#[syncbat::operation( + descriptor = CLAIMS_LIST, + register = register_claims_list, + register_item = claims_list_item, + name = "texo.claims.list", + effect = Inspect, + input_schema = "texo.claims.list.input.v3", + output_schema = "texo.claims.list.output.v3", + receipt_kind = "receipt.texo.claims.list.v3", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn claims_list(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.claims.list", || { + let input: ClaimsListInput = parse_input("texo.claims.list", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.claims.list", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + let claims = claim_list_rows(&view, input.subject.as_deref())?; + Ok(ClaimsListOutput { + workspace_id: view.workspace_id.clone(), + frontier: view.frontier, + claims, + snapshot, + }) + }) +} +#[syncbat::operation( + descriptor = CLAIMS_SEARCH, + register = register_claims_search, + register_item = claims_search_item, + name = "texo.claims.search", + effect = Inspect, + input_schema = "texo.claims.search.input.v2", + output_schema = "texo.claims.search.output.v2", + receipt_kind = "receipt.texo.claims.search.v2", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn claims_search(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.claims.search", || { + let input: ClaimsSearchInput = parse_input("texo.claims.search", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.claims.search", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + let query = input.query.unwrap_or_default(); + if query.len() > 256 { + return Err(TexoError::OpInput { + op: "texo.claims.search".to_string(), + detail: "query exceeds 256 bytes".to_string(), + }); + } + let limit = input.limit.unwrap_or(25); + if !(1..=100).contains(&limit) { + return Err(TexoError::OpInput { + op: "texo.claims.search".to_string(), + detail: "limit must be between 1 and 100".to_string(), + }); + } + let offset = parse_claim_search_cursor(input.cursor.as_deref())?; + let query_terms = query + .split_whitespace() + .map(str::to_ascii_lowercase) + .collect::>(); + let rows = claim_list_rows(&view, input.subject.as_deref())? + .into_iter() + .filter(|row| input.status.is_none_or(|status| row.status == status)) + .filter(|row| claim_matches_query(row, &query_terms)) + .collect::>(); + let total = rows.len(); + let page = rows + .into_iter() + .skip(offset) + .take(limit) + .collect::>(); + let returned = page.len(); + let next_offset = offset.saturating_add(returned); + let has_more = next_offset < total; + Ok(ClaimsSearchOutput { + workspace_id: view.workspace_id.clone(), + frontier: view.frontier, + freshness: view.freshness, + total, + returned, + has_more, + next_cursor: has_more.then(|| format!("texo-claims-v1:{next_offset}")), + claims: page, + snapshot, + }) + }) +} +#[syncbat::operation( + descriptor = KNOWLEDGE_SEARCH, + register = register_knowledge_search, + register_item = knowledge_search_item, + name = "texo.knowledge.search", + effect = Inspect, + input_schema = "texo.knowledge.search.input.v1", + output_schema = "texo.knowledge.search.output.v1", + receipt_kind = "receipt.texo.knowledge.search.v1", + reads_events = ["evt.e00e"], + queries_projections = ["texo.workspace.view.v2", "texo.code.index.v1"] +)] +#[tracing::instrument(skip_all)] +fn knowledge_search(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.knowledge.search", || { + let input: KnowledgeSearchInput = parse_input("texo.knowledge.search", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.knowledge.search", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + search_knowledge_from_view(&view, snapshot, &input) + }) +} +#[syncbat::operation( + descriptor = CLAIM_EXPLAIN, + register = register_claim_explain, + register_item = claim_explain_item, + name = "texo.claim.explain", + effect = Inspect, + input_schema = "texo.claim.explain.input.v3", + output_schema = "texo.claim.explain.output.v4", + receipt_kind = "receipt.texo.claim.explain.v4", + queries_projections = ["texo.claim.explain.v2"] +)] +#[tracing::instrument(skip_all)] +fn claim_explain(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.claim.explain", || { + let input: ClaimExplainInput = parse_input("texo.claim.explain", input)?; + cx.projection_read_handle() + .query_projection(CLAIM_EXPLAIN_PROJECTION) + .map_err(|error| op_runtime("texo.claim.explain", error))?; + let entity = entity_for_claim(&input.claim_id); + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + let card = view + .claims + .iter() + .find(|claim| claim.card.claim_id == input.claim_id) + .map(|claim| claim.card.as_ref().clone()) + .ok_or_else(|| TexoError::MissingEntity { + entity: entity.clone(), + })?; + let timeline = claim_timeline_through(&entity, view.frontier)?; + let evidence = evidence_projection_through(view.frontier)?.take_claim(&input.claim_id); + let coverage = coverage_for_view(&view, &snapshot)?; + let answer_state = answer_state_for_claim( + view.claims + .iter() + .find(|claim| claim.card.claim_id == input.claim_id) + .map(|claim| claim.status), + &evidence, + ); + Ok(ClaimExplainOutput { + card, + timeline: timeline.entries, + answer_state, + evidence, + coverage, + snapshot, + }) + }) +} +#[syncbat::operation( + descriptor = CLAIM_SUPERSEDE, + register = register_claim_supersede, + register_item = claim_supersede_item, + name = "texo.claim.supersede", + effect = Persist, + input_schema = "texo.claim.supersede.input.v2", + output_schema = "texo.claim.supersede.output.v2", + receipt_kind = "receipt.texo.claim.supersede.v2", + appends_events = ["evt.e003"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn claim_supersede(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.claim.supersede", || { + let input: ClaimSupersedeInput = parse_input("texo.claim.supersede", input)?; + if input.old == input.new { + return Err(TexoError::OpInput { + op: "texo.claim.supersede".to_string(), + detail: "old and new claims must differ".to_string(), + }); + } + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.claim.supersede", error))?; + + let old_entity = entity_for_claim(&input.old); + let new_entity = entity_for_claim(&input.new); + let (old_card, new_card, workspace_id) = env::with(|op_env| { + let old_card = env::deterministic_projection(|| { + op_env + .store + .project::(&old_entity, &Freshness::Consistent) + })?; + let new_card = env::deterministic_projection(|| { + op_env + .store + .project::(&new_entity, &Freshness::Consistent) + })?; + Ok::<_, TexoError>((old_card, new_card, op_env.workspace_id.clone())) + })??; + let old_card = old_card.ok_or_else(|| TexoError::MissingEntity { + entity: old_entity.clone(), + })?; + let _new_card = new_card.ok_or_else(|| TexoError::MissingEntity { + entity: new_entity.clone(), + })?; + if old_card.phase == 2 && old_card.superseded_by.as_deref() == Some(input.new.as_str()) { + return Ok(ClaimSupersedeOutput { + old: input.old, + new: input.new, + already_applied: true, + receipt: None, + }); + } + if old_card.phase != 1 { + return Err(TexoError::Transition { + machine: CLAIM_MACHINE.to_string(), + from: old_card.phase, + to: 2, + context: Some(format!( + "claim {} is already {}{}", + input.old, + claim_phase_name(old_card.phase), + old_card + .superseded_by + .as_deref() + .map_or_else(String::new, |successor| format!(" by {successor}")) + )), + }); + } + + let payload = ClaimSupersededV2 { + old_claim_id: input.old.clone(), + new_claim_id: input.new.clone(), + workspace_id, + reason: input.reason, + decided_by: input.decided_by, + observed_at_ms: input.observed_at_ms, + transition: transition_record( + CLAIM_MACHINE, + &old_entity, + 1, + 2, + vec![TransitionCauseV1 { + lane: 0, + key: format!("claim:{}", input.new), + }], + input.observed_at_ms, + ), + }; + append_json( + "texo.claim.supersede", + cx, + ::KIND, + &payload, + )?; + Ok(ClaimSupersedeOutput { + old: input.old, + new: input.new, + already_applied: false, + receipt: Some(take_one_receipt("texo.claim.supersede")?), + }) + }) +} +#[derive(Debug, Deserialize)] +struct ClaimsListInput { + subject: Option, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Deserialize)] +struct ClaimsSearchInput { + query: Option, + subject: Option, + status: Option, + limit: Option, + cursor: Option, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +struct ClaimsListOutput { + workspace_id: String, + frontier: u64, + claims: Vec, + snapshot: SnapshotRead, +} + +#[derive(Debug, Serialize)] +struct ClaimsSearchOutput { + workspace_id: String, + frontier: u64, + freshness: crate::claims::workspace::ProjectionFreshness, + total: usize, + returned: usize, + has_more: bool, + next_cursor: Option, + claims: Vec, + snapshot: SnapshotRead, +} + +#[derive(Debug, Deserialize)] +struct KnowledgeSearchInput { + #[serde(default)] + query: Option, + #[serde(default)] + subject: Option, + #[serde(default)] + status: Option, + #[serde(default)] + limit: Option, + #[serde(default)] + cursor: Option, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +struct KnowledgeSearchOutput { + workspace_id: String, + frontier: u64, + total: usize, + returned: usize, + has_more: bool, + next_cursor: Option, + results: Vec, + code_index_available: bool, + coverage: KnowledgeCoverage, + snapshot: SnapshotRead, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum KnowledgeSearchResult { + Claim { claim: AgentClaimRow }, + Code { occurrence: CodeOccurrence }, +} + +struct RankedKnowledgeResult { + rank: u8, + key: String, + result: KnowledgeSearchResult, +} + +#[derive(Debug, Deserialize)] +struct ClaimExplainInput { + claim_id: String, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +struct ClaimExplainOutput { + card: ClaimCard, + timeline: Vec, + answer_state: AnswerState, + evidence: Vec, + coverage: KnowledgeCoverage, + snapshot: SnapshotRead, +} +#[derive(Debug, Deserialize)] +struct ClaimSupersedeInput { + old: String, + new: String, + reason: String, + decided_by: String, + observed_at_ms: u64, +} + +#[derive(Debug, Serialize)] +struct ClaimSupersedeOutput { + old: String, + new: String, + already_applied: bool, + receipt: Option, +} + +pub(super) fn claim_list_rows( + view: &WorkspaceView, + subject: Option<&str>, +) -> Result, TexoError> { + let mut rows = Vec::new(); + // One index scan for all receipts instead of one by_entity per claim. + let receipts = claim_record_receipts()?; + for claim in &view.claims { + if subject.is_some_and(|wanted| claim.card.subject_hint.as_deref() != Some(wanted)) { + continue; + } + let receipt = match receipts.get(&claim.card.claim_id) { + Some(row) => row.clone(), + None => claim_receipt(&claim.card.claim_id)?, + }; + rows.push(AgentClaimRow { + claim_id: claim.card.claim_id.clone(), + status: claim.status, + subject_hint: claim.card.subject_hint.clone(), + text: claim.card.text.clone(), + source: AgentSourceRow { + source_id: claim.card.source_id.clone(), + path: claim.card.source_path.clone(), + line_start: claim.card.line_start, + }, + receipt, + supersedes: claim.supersedes.clone(), + superseded_by: claim.card.superseded_by.clone(), + }); + } + Ok(rows) +} + +fn search_knowledge_from_view( + view: &WorkspaceView, + snapshot: SnapshotRead, + input: &KnowledgeSearchInput, +) -> Result { + let (query, limit) = validated_knowledge_search(input)?; + let cursor_identity = knowledge_search_identity(&snapshot, query, input); + let offset = parse_knowledge_search_cursor(input.cursor.as_deref(), &cursor_identity)?; + let query_lower = query.to_ascii_lowercase(); + let mut ranked = claim_list_rows(view, input.subject.as_deref())? + .into_iter() + .filter(|claim| input.status.is_none_or(|status| claim.status == status)) + .filter_map(|claim| { + claim_search_rank(&claim, &query_lower).map(|rank| RankedKnowledgeResult { + rank, + key: format!("claim:{}", claim.claim_id), + result: KnowledgeSearchResult::Claim { claim }, + }) + }) + .collect::>(); + let mut coverage = coverage_for_view(view, &snapshot)?; + // A subject/status filter searches only the claim plane; the code plane is + // deliberately not consulted, which is "not searched", not "unavailable". + let code_searched = input.subject.is_none() && input.status.is_none(); + let mut code_index_available = false; + if code_searched { + if let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() { + let loaded = load_code_artifact_at(view.frontier, source_snapshot_id)?; + if let Some(code_coverage) = &loaded.coverage { + merge_coverage(&mut coverage, code_coverage); + } + if let Some(artifact) = loaded.artifact { + code_index_available = true; + ranked.extend(artifact.occurrences.into_iter().filter_map(|occurrence| { + code_search_rank(&occurrence, &query_lower).map(|rank| RankedKnowledgeResult { + rank, + key: format!( + "code:{}:{}:{}", + occurrence.symbol, occurrence.path, occurrence.byte_range.start + ), + result: KnowledgeSearchResult::Code { occurrence }, + }) + })); + } + } + } + // Only assert the code index is unavailable when it was actually searched and + // found absent — never for a filtered search that skipped it by design. + if code_searched + && !code_index_available + && !coverage + .gaps + .iter() + .any(|gap| gap.kind == CoverageGapKind::CodeIndexUnavailable) + { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::CodeIndexUnavailable, + }); + } + ranked.sort_by(|left, right| { + left.rank + .cmp(&right.rank) + .then_with(|| left.key.cmp(&right.key)) + }); + let total = ranked.len(); + let results = ranked + .into_iter() + .skip(offset) + .take(limit) + .map(|ranked| ranked.result) + .collect::>(); + let returned = results.len(); + let next_offset = offset.saturating_add(returned); + let has_more = next_offset < total; + Ok(KnowledgeSearchOutput { + workspace_id: view.workspace_id.clone(), + frontier: view.frontier, + total, + returned, + has_more, + next_cursor: has_more.then(|| format!("texo-knowledge-v1:{cursor_identity}:{next_offset}")), + results, + code_index_available, + coverage, + snapshot, + }) +} + +fn validated_knowledge_search(input: &KnowledgeSearchInput) -> Result<(&str, usize), TexoError> { + let query = input.query.as_deref().unwrap_or(""); + if query.len() > 256 { + return Err(TexoError::OpInput { + op: "texo.knowledge.search".to_string(), + detail: "query exceeds 256 bytes".to_string(), + }); + } + let limit = input.limit.unwrap_or(25); + if !(1..=100).contains(&limit) { + return Err(TexoError::OpInput { + op: "texo.knowledge.search".to_string(), + detail: "limit must be between 1 and 100".to_string(), + }); + } + Ok((query, limit)) +} + +fn knowledge_search_identity( + snapshot: &SnapshotRead, + query: &str, + input: &KnowledgeSearchInput, +) -> String { + let material = format!( + "texo.knowledge.search.v1\u{1f}{}\u{1f}{query}\u{1f}{}\u{1f}{}", + snapshot.token.as_str(), + input.subject.as_deref().unwrap_or(""), + input + .status + .map_or("", crate::claims::status::ClaimStatus::as_str) + ); + blake3::hash(material.as_bytes()).to_hex()[..16].to_string() +} + +fn parse_knowledge_search_cursor( + cursor: Option<&str>, + expected_identity: &str, +) -> Result { + let Some(cursor) = cursor else { + return Ok(0); + }; + let Some(rest) = cursor.strip_prefix("texo-knowledge-v1:") else { + return Err(invalid_knowledge_cursor()); + }; + let Some((identity, offset)) = rest.split_once(':') else { + return Err(invalid_knowledge_cursor()); + }; + if identity != expected_identity { + return Err(invalid_knowledge_cursor()); + } + offset.parse().map_err(|_| invalid_knowledge_cursor()) +} + +fn invalid_knowledge_cursor() -> TexoError { + TexoError::OpInput { + op: "texo.knowledge.search".to_string(), + detail: "cursor is invalid for this query and snapshot".to_string(), + } +} + +fn claim_search_rank(claim: &AgentClaimRow, query: &str) -> Option { + if query.is_empty() { + return Some(0); + } + let text = claim.text.to_ascii_lowercase(); + let subject = claim + .subject_hint + .as_deref() + .unwrap_or("") + .to_ascii_lowercase(); + let path = claim.source.path.to_ascii_lowercase(); + if text == query || subject == query { + Some(0) + } else if text.starts_with(query) || subject.starts_with(query) { + Some(1) + } else if text.contains(query) || subject.contains(query) { + Some(2) + } else if path.contains(query) { + Some(3) + } else { + None + } +} + +fn code_search_rank(occurrence: &CodeOccurrence, query: &str) -> Option { + if query.is_empty() { + return Some(1); + } + let name = occurrence.display_name.to_ascii_lowercase(); + let symbol = occurrence.symbol.to_ascii_lowercase(); + let path = occurrence.path.to_ascii_lowercase(); + if name == query || symbol == query { + Some(0) + } else if name.starts_with(query) { + Some(1) + } else if name.contains(query) || symbol.contains(query) { + Some(2) + } else if path.contains(query) { + Some(3) + } else { + None + } +} + +pub(super) fn parse_claim_search_cursor(cursor: Option<&str>) -> Result { + let Some(cursor) = cursor else { + return Ok(0); + }; + let Some(offset) = cursor.strip_prefix("texo-claims-v1:") else { + return Err(TexoError::OpInput { + op: "texo.claims.search".to_string(), + detail: "cursor has an unsupported schema".to_string(), + }); + }; + offset.parse::().map_err(|_| TexoError::OpInput { + op: "texo.claims.search".to_string(), + detail: "cursor offset is invalid".to_string(), + }) +} + +pub(super) fn claim_matches_query(row: &AgentClaimRow, terms: &[String]) -> bool { + if terms.is_empty() { + return true; + } + let mut searchable = row.text.to_ascii_lowercase(); + searchable.push(' '); + searchable.push_str(&row.source.path.to_ascii_lowercase()); + if let Some(subject) = &row.subject_hint { + searchable.push(' '); + searchable.push_str(&subject.to_ascii_lowercase()); + } + terms.iter().all(|term| searchable.contains(term)) +} diff --git a/src/ops/handlers/common.rs b/src/ops/handlers/common.rs new file mode 100644 index 0000000..7c540ed --- /dev/null +++ b/src/ops/handlers/common.rs @@ -0,0 +1,550 @@ +use super::knowledge_read::{load_code_artifact_at, merge_coverage}; +use super::model::AgentReceiptRow; +use crate::claims::evidence::{assemble_through as assemble_evidence_through, EvidenceProjection}; +use crate::claims::temporal::{assemble_through as assemble_temporal_through, TemporalProjection}; +use crate::claims::timeline::ClaimTimeline; +use crate::claims::workspace::{assemble, assemble_through, WorkspaceView}; +use crate::error::{SnapshotFailureKind, TexoError}; +use crate::events::coordinate::{entity_for_claim, scope_for_workspace}; +use crate::events::ids::{ClaimId, WorkspaceId}; +use crate::events::payloads::{ + ClaimRecordedV2, SourceSnapshotRecordedV1, SourceSnapshotRelationV1, +}; +use crate::git_source::{compare_commits, CaptureLimits}; +use crate::knowledge::{ + AnalysisQuality, CoverageGap, CoverageGapKind, EvidenceLinkMethod, EvidenceStance, + KnowledgeCoverage, SnapshotDescriptor, SnapshotRead, SnapshotToken, TemporalRelation, +}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use crate::semantics::pipeline::RelateTemporalPolicy; +use batpak::coordinate::Region; +use batpak::event::EventSourced; +use batpak::event::{EventKind, EventPayload}; +use batpak::id::EntityIdType; +use serde::Serialize; +use std::collections::BTreeMap; +use std::path::Path; +use std::time::Instant; +use syncbat::{HandlerError, HandlerResult}; + +pub(super) const WORKSPACE_VIEW_PROJECTION: &str = "texo.workspace.view.v2"; +const MAX_TEMPORAL_SNAPSHOT_COMPARISONS: usize = 1_024; +const MAX_GIT_ANCESTRY_WALK: usize = 100_000; + +pub(super) fn elapsed_ms(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +pub(crate) fn run_op( + op: &'static str, + f: impl FnOnce() -> Result, +) -> HandlerResult { + let output = f()?; + batpak::canonical::to_bytes(&output).map_err(|error| { + HandlerError::from(TexoError::OpRuntime { + op: op.to_string(), + detail: error.to_string(), + denied: false, + }) + }) +} + +pub(crate) fn parse_input( + op: &str, + input: &[u8], +) -> Result { + batpak::canonical::from_bytes(input).map_err(|error| TexoError::OpInput { + op: op.to_string(), + detail: error.to_string(), + }) +} + +pub(crate) fn append_json( + op: &str, + cx: &mut syncbat::Ctx<'_>, + kind: EventKind, + payload: &T, +) -> Result<(), TexoError> { + let bytes = batpak::canonical::to_bytes(payload).map_err(|error| TexoError::OpRuntime { + op: op.to_string(), + detail: format!("canonical effect payload encoding failed: {error}"), + denied: false, + })?; + cx.event_append_handle() + .append_event(kind, &bytes) + .map_err(|error| op_runtime(op, error)) +} + +pub(crate) fn take_receipts() -> Result, TexoError> { + env::with(|op_env| op_env.receipts.borrow_mut().drain(..).collect()) +} + +pub(crate) fn take_one_receipt(op: &str) -> Result { + let mut receipts = take_receipts()?; + receipts.pop().ok_or_else(|| TexoError::OpRuntime { + op: op.to_string(), + detail: "append produced no receipt".to_string(), + denied: false, + }) +} + +pub(crate) fn op_runtime(op: &str, error: impl std::fmt::Display) -> TexoError { + TexoError::OpRuntime { + op: op.to_string(), + detail: error.to_string(), + denied: false, + } +} + +pub(crate) fn config_error(error: crate::config::ConfigError) -> TexoError { + TexoError::Config { + detail: error.to_string(), + source: Some(Box::new(error)), + } +} + +pub(crate) fn assemble_current_view() -> Result, TexoError> { + env::with(|op_env| { + let mut cache = op_env.cache.borrow_mut(); + env::deterministic_projection(|| assemble(&op_env.store, &op_env.workspace_id, &mut cache)) + })? +} + +pub(crate) fn assemble_snapshot_view( + requested: Option<&str>, +) -> Result<(std::sync::Arc, SnapshotRead), TexoError> { + let Some(requested) = requested else { + let view = assemble_current_view()?; + let snapshot = snapshot_for_view(&view)?; + return Ok((view, snapshot)); + }; + let (store, workspace, journal_id) = env::with(|op_env| { + ( + op_env.store.clone(), + op_env.workspace_id.clone(), + op_env.journal.id.clone(), + ) + })?; + let workspace_id = WorkspaceId::new(workspace.clone())?; + let descriptor = SnapshotToken::resolve_for_journal(requested, &workspace_id, &journal_id) + .map_err(|error| TexoError::Snapshot { + kind: SnapshotFailureKind::InvalidToken, + detail: error.to_string(), + })?; + let available_source = + latest_source_snapshot(Some(descriptor.frontier))?.map(|snapshot| snapshot.snapshot_id); + if available_source != descriptor.source_snapshot_id { + return Err(TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: "the token's source snapshot is unavailable at its journal frontier" + .to_string(), + }); + } + validate_snapshot_anchor(&store, &workspace, &descriptor)?; + let view = + env::deterministic_projection(|| assemble_through(&store, &workspace, descriptor.frontier)) + .map_err(|error| TexoError::Snapshot { + kind: SnapshotFailureKind::Unavailable, + detail: error.to_string(), + })?; + Ok((view, SnapshotRead::new(descriptor))) +} + +pub(crate) fn snapshot_for_view(view: &WorkspaceView) -> Result { + let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; + let (anchor_event_id_hex, journal_id) = env::with(|op_env| { + Ok::<_, TexoError>(( + anchor_at_frontier(&op_env.store, &op_env.workspace_id, view.frontier)?, + op_env.journal.id.clone(), + )) + })??; + let source_snapshot_id = + latest_source_snapshot(Some(view.frontier))?.map(|snapshot| snapshot.snapshot_id); + Ok(SnapshotRead::new(SnapshotDescriptor { + workspace_id, + journal_id, + frontier: view.frontier, + anchor_event_id_hex, + source_snapshot_id, + })) +} + +pub(crate) fn validate_snapshot_anchor( + store: &crate::journal_store::JournalStore, + workspace_id: &str, + descriptor: &SnapshotDescriptor, +) -> Result<(), TexoError> { + let actual = anchor_at_frontier(store, workspace_id, descriptor.frontier)?; + if actual != descriptor.anchor_event_id_hex { + return Err(TexoError::Snapshot { + kind: SnapshotFailureKind::AnchorMismatch, + detail: format!( + "journal anchor at frontier {} differs from the token", + descriptor.frontier + ), + }); + } + Ok(()) +} + +pub(crate) fn anchor_at_frontier( + store: &crate::journal_store::JournalStore, + workspace_id: &str, + frontier: u64, +) -> Result { + if frontier == 0 { + return Ok(String::new()); + } + let region = Region::scope(scope_for_workspace(workspace_id)); + let entry = store + .query_entries_after(®ion, Some(frontier.saturating_sub(1)), 1) + .into_iter() + .next() + .filter(|entry| entry.global_sequence() == frontier) + .ok_or_else(|| TexoError::Snapshot { + kind: SnapshotFailureKind::Unavailable, + detail: format!("workspace frontier {frontier} is unavailable"), + })?; + Ok(format!("{:032x}", entry.event_id().as_u128())) +} + +pub(crate) fn claim_timeline_through( + entity: &str, + frontier: u64, +) -> Result { + env::with(|op_env| { + let mut timeline = ClaimTimeline::default(); + for entry in op_env.store.by_entity(entity) { + if entry.global_sequence() > frontier { + break; + } + let raw = op_env.store.read_raw(entry.event_id())?; + timeline.apply_event(&raw.event); + } + Ok::<_, TexoError>(timeline) + })? +} + +pub(crate) fn coverage_for_view( + view: &WorkspaceView, + snapshot: &SnapshotRead, +) -> Result { + // Only a genuinely missing snapshot (`Ok(None)`) or a mismatched identity + // degrades to `Unavailable`; a decode/corruption error must bubble up rather + // than masquerade as an ordinary absent snapshot. + if let Some(recorded) = latest_source_snapshot(Some(view.frontier))? { + if Some(&recorded.snapshot_id) == snapshot.descriptor.source_snapshot_id.as_ref() { + return Ok(recorded.coverage); + } + } + Ok(KnowledgeCoverage { + analysis_quality: AnalysisQuality::Unavailable, + sources_examined: u64::try_from(view.sources.len()).unwrap_or(u64::MAX), + occurrences: u64::try_from(view.claims.len()).unwrap_or(u64::MAX), + truncated: false, + gaps: vec![CoverageGap { + path: None, + kind: CoverageGapKind::SourceSnapshotUnavailable, + }], + }) +} + +pub(crate) fn status_coverage( + view: &WorkspaceView, + snapshot: &SnapshotRead, +) -> Result<(KnowledgeCoverage, bool), TexoError> { + let mut coverage = coverage_for_view(view, snapshot)?; + let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() else { + return Ok((coverage, false)); + }; + let loaded = load_code_artifact_at(view.frontier, source_snapshot_id)?; + if let Some(code_coverage) = loaded.coverage { + merge_coverage(&mut coverage, &code_coverage); + } + if loaded.unavailable { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::CodeIndexUnavailable, + }); + } + Ok((coverage, !loaded.unavailable)) +} + +/// Reconstruct the capture bounds a snapshot was recorded with so code indexing +/// recaptures the same bounded world instead of the defaults. +pub(crate) fn recorded_capture_limits(recorded: &SourceSnapshotRecordedV1) -> CaptureLimits { + CaptureLimits { + max_files: usize::try_from(recorded.capture_max_files).unwrap_or(usize::MAX), + max_file_bytes: recorded.capture_max_file_bytes, + max_total_bytes: recorded.capture_max_total_bytes, + } +} + +pub(crate) fn latest_source_snapshot( + frontier: Option, +) -> Result, TexoError> { + Ok(source_snapshots_through(frontier)?.pop()) +} + +/// The recorded snapshot with this content-addressed id, regardless of whether a +/// newer snapshot was recorded afterward (revisiting a prior commit re-derives an +/// existing id rather than appending a new record). +pub(crate) fn source_snapshot_by_id( + snapshot_id: &crate::knowledge::SourceSnapshotId, +) -> Result, TexoError> { + Ok(source_snapshots_through(None)? + .into_iter() + .rev() + .find(|snapshot| &snapshot.snapshot_id == snapshot_id)) +} + +pub(crate) fn source_snapshots_through( + frontier: Option, +) -> Result, TexoError> { + env::with(|op_env| { + let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); + let mut after = None; + let mut snapshots = Vec::new(); + 'pages: loop { + let page = op_env.store.query_entries_after(®ion, after, 256); + if page.is_empty() { + break; + } + for entry in &page { + if frontier.is_some_and(|frontier| entry.global_sequence() > frontier) { + break 'pages; + } + if entry.event_kind() == ::KIND { + let raw = op_env.store.read_raw(entry.event_id())?; + snapshots.push( + batpak::encoding::from_bytes::( + &raw.event.payload, + ) + .map_err(|error| TexoError::Decode { + entity: entry.coord().entity().to_string(), + detail: error.to_string(), + })?, + ); + } + } + after = page.last().map(batpak::store::IndexEntry::global_sequence); + } + Ok::<_, TexoError>(snapshots) + })? +} + +pub(crate) fn plan_snapshot_relations( + root: &Path, + workspace_id: &WorkspaceId, + capture: &crate::git_source::GitCapture, + previous: &[SourceSnapshotRecordedV1], + observed_at_ms: u64, +) -> Result<(Vec, Vec), TexoError> { + let skipped = previous + .len() + .saturating_sub(MAX_TEMPORAL_SNAPSHOT_COMPARISONS); + let mut gaps = Vec::new(); + if skipped > 0 { + gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::BudgetExceeded, + }); + } + let mut relations = Vec::new(); + for prior in previous.iter().skip(skipped) { + if prior.snapshot_id == capture.snapshot_id { + continue; + } + let comparison = if prior.repository_id == capture.repository_id { + overlay_aware_comparison(root, prior, capture)? + } else { + crate::git_source::GitComparison { + relation: TemporalRelation::Unknown, + gap: Some(CoverageGapKind::MissingObject), + } + }; + if let Some(kind) = comparison.gap { + let gap = CoverageGap { path: None, kind }; + if !gaps.contains(&gap) { + gaps.push(gap); + } + } + // Never journal an `Unknown` ordering: the relation idempotency key is + // (workspace, left, right) and replay keeps the first fact, so a durable + // Unknown from shallow history or an exhausted walk would permanently + // shadow the real Before/After discoverable once full history arrives. + // Its absence already reads as Unknown, and the gap above records why. + if comparison.relation == TemporalRelation::Unknown { + continue; + } + relations.push(SourceSnapshotRelationV1 { + workspace_id: workspace_id.clone(), + repository_id: capture.repository_id.clone(), + left_snapshot_id: prior.snapshot_id.clone(), + right_snapshot_id: capture.snapshot_id.clone(), + left_commit: prior.base_commit.clone(), + right_commit: capture.base_commit.clone(), + relation: comparison.relation, + observed_at_ms, + }); + } + Ok((relations, gaps)) +} + +pub(crate) fn overlay_aware_comparison( + root: &Path, + prior: &SourceSnapshotRecordedV1, + capture: &crate::git_source::GitCapture, +) -> Result { + use crate::git_source::GitComparison; + + if prior.base_commit == capture.base_commit { + return Ok(GitComparison { + relation: match (prior.dirty, capture.dirty) { + (false, true) => TemporalRelation::Before, + (true, false) => TemporalRelation::After, + (true, true) => TemporalRelation::Concurrent, + (false, false) => TemporalRelation::Same, + }, + gap: None, + }); + } + let comparison = compare_commits( + root, + &prior.base_commit, + &capture.base_commit, + MAX_GIT_ANCESTRY_WALK, + )?; + let relation = match comparison.relation { + TemporalRelation::Before if prior.dirty => TemporalRelation::Concurrent, + TemporalRelation::After if capture.dirty => TemporalRelation::Concurrent, + TemporalRelation::Same => TemporalRelation::Same, + TemporalRelation::Before => TemporalRelation::Before, + TemporalRelation::After => TemporalRelation::After, + TemporalRelation::Concurrent => TemporalRelation::Concurrent, + TemporalRelation::Unknown => TemporalRelation::Unknown, + }; + Ok(GitComparison { + relation, + gap: comparison.gap, + }) +} + +pub(crate) fn plan_and_attach_snapshot_relations( + root: &Path, + workspace_id: &WorkspaceId, + capture: &mut crate::git_source::GitCapture, + previous: &[SourceSnapshotRecordedV1], + observed_at_ms: u64, +) -> Result, TexoError> { + let (relations, gaps) = + plan_snapshot_relations(root, workspace_id, capture, previous, observed_at_ms)?; + for gap in gaps { + if capture.coverage.gaps.len() < 256 && !capture.coverage.gaps.contains(&gap) { + capture.coverage.gaps.push(gap); + } + } + Ok(relations) +} + +pub(crate) fn evidence_projection_through(frontier: u64) -> Result { + env::with(|op_env| { + env::deterministic_projection(|| { + assemble_evidence_through(&op_env.store, &op_env.workspace_id, frontier) + }) + })? +} + +pub(crate) fn temporal_projection_through(frontier: u64) -> Result { + env::with(|op_env| { + env::deterministic_projection(|| { + assemble_temporal_through(&op_env.store, &op_env.workspace_id, frontier) + }) + })? +} + +pub(crate) fn workspace_temporal_policy( + view: &WorkspaceView, +) -> Result { + workspace_temporal_policy_through(view, view.frontier) +} + +pub(crate) fn workspace_temporal_policy_through( + view: &WorkspaceView, + frontier: u64, +) -> Result { + let evidence = evidence_projection_through(frontier)?; + let relations = temporal_projection_through(frontier)?; + let mut policy = RelateTemporalPolicy::default(); + for claim in &view.claims { + let claim_id = ClaimId::try_from(claim.card.claim_id.as_str())?; + if let Some(latest) = evidence + .for_claim(claim_id.as_str()) + .iter() + .filter(|item| { + item.method == EvidenceLinkMethod::Deterministic + && item.stance == EvidenceStance::Supports + }) + .max_by_key(|item| item.link_sequence) + { + policy.bind_claim(&claim_id, &latest.occurrence.snapshot_id); + } + } + for (left, right, relation) in relations.facts() { + policy.insert_relation_ids(left, right, relation); + } + Ok(policy) +} + +pub(crate) fn semantic_temporal_policy( + view: &WorkspaceView, +) -> Result { + workspace_temporal_policy(view) +} +pub(crate) fn claim_record_receipts() -> Result, TexoError> { + env::with(|op_env| { + let scope = scope_for_workspace(&op_env.workspace_id); + let mut receipts = BTreeMap::new(); + for entry in op_env.store.by_scope(&scope) { + if entry.event_kind() != ::KIND { + continue; + } + // The entity coordinate is "claim:{claim_id}" — the payload decode + // it used to do here (read_raw + msgpack per event) carried no + // information the index entry does not already hold. + let Some(claim_id) = entry.coord().entity().strip_prefix("claim:") else { + continue; + }; + receipts + .entry(claim_id.to_string()) + .or_insert(AgentReceiptRow { + event_id: event_id_hex(entry.event_id()), + sequence: entry.global_sequence(), + }); + } + Ok::<_, TexoError>(receipts) + })? +} + +fn event_id_hex(event_id: batpak::id::EventId) -> String { + format!("{:032x}", event_id.as_u128()) +} + +pub(crate) fn claim_receipt(claim_id: &str) -> Result { + let entity = entity_for_claim(claim_id); + env::with(|op_env| { + let entry = op_env + .store + .by_entity(&entity) + .into_iter() + .find(|entry| entry.event_kind() == ::KIND) + .ok_or_else(|| TexoError::MissingEntity { + entity: entity.clone(), + })?; + Ok::<_, TexoError>(AgentReceiptRow { + event_id: event_id_hex(entry.event_id()), + sequence: entry.global_sequence(), + }) + })? +} diff --git a/src/ops/handlers/compile.rs b/src/ops/handlers/compile.rs new file mode 100644 index 0000000..3c3c903 --- /dev/null +++ b/src/ops/handlers/compile.rs @@ -0,0 +1,137 @@ +use super::agent_context::{build_agent_context_from_view, AgentContextOutput}; +use super::common::{ + append_json, assemble_current_view, op_runtime, parse_input, run_op, snapshot_for_view, + take_one_receipt, WORKSPACE_VIEW_PROJECTION, +}; +use super::ingest::resolve_path; +use super::relate::require_complete_settlement; +use super::render::{self, StalenessReport}; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::payloads::OnboardingCompiledV2; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use crate::relate::heuristic; +use batpak::event::EventPayload; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = COMPILE_RUN, + register = register_compile_run, + register_item = compile_run_item, + name = "texo.compile.run", + effect = Persist, + input_schema = "texo.compile.run.input.v3", + output_schema = "texo.compile.run.output.v2", + receipt_kind = "receipt.texo.compile.run.v2", + appends_events = ["evt.e005"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn compile_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.compile.run", || { + let input: CompileRunInput = parse_input("texo.compile.run", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.compile.run", error))?; + let view = assemble_current_view()?; + if !input.allow_unsettled { + require_complete_settlement(&view)?; + } + let snapshot = snapshot_for_view(&view)?; + let context = build_agent_context_from_view(&view, None, true, snapshot.clone())?; + let conflict_report = heuristic::detect_conflicts(&view)?; + let (root, workspace_id) = + env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; + let out_dir = resolve_path(&root, &input.out_dir); + let stale_report = StalenessReport::empty(workspace_id.clone(), view.frontier, snapshot); + let files = compile_artifacts(&context, &view, &stale_report, &conflict_report)?; + for file in &files { + let path = out_dir.join(&file.name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, file.contents.as_bytes())?; + } + let source_claim_ids = view + .claims + .iter() + .map(|claim| claim.card.claim_id.clone()) + .collect::>(); + let doc_id = format!( + "doc_{}", + &blake3::hash(out_dir.to_string_lossy().as_bytes()).to_hex()[..12] + ); + append_json( + "texo.compile.run", + cx, + ::KIND, + &OnboardingCompiledV2 { + doc_id, + workspace_id, + output_path: input.out_dir.to_string_lossy().to_string(), + source_claim_ids, + replayed_through_sequence: view.frontier, + compiled_at_ms: input.observed_at_ms, + }, + )?; + Ok(CompileRunOutput { + files: files.into_iter().map(|file| file.name).collect::>(), + receipt: take_one_receipt("texo.compile.run")?, + }) + }) +} +#[derive(Debug, Deserialize)] +struct CompileRunInput { + out_dir: PathBuf, + observed_at_ms: u64, + #[serde(default)] + allow_unsettled: bool, +} + +#[derive(Debug, Serialize)] +struct CompileRunOutput { + files: Vec, + receipt: ReceiptNote, +} + +struct CompileFile { + name: String, + contents: String, +} + +fn compile_artifacts( + context: &AgentContextOutput, + view: &WorkspaceView, + stale: &StalenessReport, + conflicts: &heuristic::ConflictReport, +) -> Result, TexoError> { + Ok(vec![ + CompileFile { + name: "onboarding.generated.md".to_string(), + contents: render::render_onboarding(context), + }, + CompileFile { + name: "claims.json".to_string(), + contents: serde_json::to_string_pretty(view)?, + }, + CompileFile { + name: "stale-context.json".to_string(), + contents: serde_json::to_string_pretty(stale)?, + }, + CompileFile { + name: "conflicts.json".to_string(), + contents: serde_json::to_string_pretty(conflicts)?, + }, + CompileFile { + name: "agent-context.json".to_string(), + contents: serde_json::to_string_pretty(context)?, + }, + CompileFile { + name: "index.html".to_string(), + contents: render::render_index_html(context, stale, conflicts)?, + }, + ]) +} diff --git a/src/ops/handlers/conflicts.rs b/src/ops/handlers/conflicts.rs new file mode 100644 index 0000000..c3941cd --- /dev/null +++ b/src/ops/handlers/conflicts.rs @@ -0,0 +1,314 @@ +use super::common::{ + append_json, assemble_current_view, op_runtime, parse_input, run_op, take_one_receipt, + WORKSPACE_VIEW_PROJECTION, +}; +use super::stats::conflict_phase_name; +use crate::claims::card::ClaimCard; +use crate::claims::conflict::ConflictCard; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::coordinate::{entity_for_claim, entity_for_conflict}; +use crate::events::machines::{transition_record, TransitionCauseV1, CONFLICT_MACHINE}; +use crate::events::payloads::{ConflictOpenedV2, ConflictResolvedV2}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use crate::relate::heuristic; +use batpak::event::EventPayload; +use batpak::store::Freshness; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = CONFLICTS_LIST, + register = register_conflicts_list, + register_item = conflicts_list_item, + name = "texo.conflicts.list", + effect = Inspect, + input_schema = "texo.conflicts.list.input.v2", + output_schema = "texo.conflicts.list.output.v2", + receipt_kind = "receipt.texo.conflicts.list.v2", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +pub(super) fn conflicts_list(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.conflicts.list", || { + let _input: ConflictsListInput = parse_input("texo.conflicts.list", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.conflicts.list", error))?; + let view = assemble_current_view()?; + Ok(conflicts_output(&view)) + }) +} +#[syncbat::operation( + descriptor = CONFLICTS_COMMIT, + register = register_conflicts_commit, + register_item = conflicts_commit_item, + name = "texo.conflicts.commit", + effect = Persist, + input_schema = "texo.conflicts.commit.input.v2", + output_schema = "texo.conflicts.commit.output.v2", + receipt_kind = "receipt.texo.conflicts.commit.v2", + appends_events = ["evt.e004"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +pub(super) fn conflicts_commit(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.conflicts.commit", || { + let input: ConflictsCommitInput = parse_input("texo.conflicts.commit", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.conflicts.commit", error))?; + let view = assemble_current_view()?; + let detected = heuristic::detect_conflicts(&view)?; + let existing = view + .conflicts + .iter() + .map(|conflict| conflict.conflict_id.clone()) + .collect::>(); + let mut committed = Vec::new(); + for entry in detected.conflicts { + if existing.contains(&entry.conflict_id) { + continue; + } + let newer = newer_claim(&view, &entry.claim_a, &entry.claim_b)?; + append_json( + "texo.conflicts.commit", + cx, + ::KIND, + &ConflictOpenedV2 { + conflict_id: entry.conflict_id.clone(), + workspace_id: view.workspace_id.clone(), + claim_a: entry.claim_a.clone(), + claim_b: entry.claim_b.clone(), + reason: entry.reason.clone(), + detector: "heuristic-v1".to_string(), + observed_at_ms: input.observed_at_ms, + transition: transition_record( + CONFLICT_MACHINE, + &entity_for_conflict(&entry.conflict_id), + 0, + 1, + vec![TransitionCauseV1 { + lane: 0, + key: format!("ingest:{}", newer.source_id), + }], + input.observed_at_ms, + ), + }, + )?; + let receipt = take_one_receipt("texo.conflicts.commit")?; + committed.push(CommittedConflict { + conflict_id: entry.conflict_id, + sequence: receipt.global_sequence, + receipt, + }); + } + Ok(committed) + }) +} +#[syncbat::operation( + descriptor = CONFLICT_RESOLVE, + register = register_conflict_resolve, + register_item = conflict_resolve_item, + name = "texo.conflict.resolve", + effect = Persist, + input_schema = "texo.conflict.resolve.input.v2", + output_schema = "texo.conflict.resolve.output.v2", + receipt_kind = "receipt.texo.conflict.resolve.v2", + appends_events = ["evt.e006"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +pub(super) fn conflict_resolve(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.conflict.resolve", || { + let input: ConflictResolveInput = parse_input("texo.conflict.resolve", input)?; + if input.resolution != "resolved" && input.resolution != "ignored" { + return Err(TexoError::StatusParse { + value: input.resolution, + }); + } + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.conflict.resolve", error))?; + let entity = entity_for_conflict(&input.conflict_id); + let card = env::with(|op_env| { + env::deterministic_projection(|| { + op_env + .store + .project::(&entity, &Freshness::Consistent) + }) + })??; + let card = card.ok_or_else(|| TexoError::MissingEntity { + entity: entity.clone(), + })?; + let target_phase = if input.resolution == "resolved" { 2 } else { 3 }; + if card.phase == target_phase { + return Ok(ConflictResolveOutput { + conflict_id: input.conflict_id, + resolution: input.resolution, + already_applied: true, + receipt: None, + }); + } + if card.phase != 1 { + return Err(TexoError::Transition { + machine: CONFLICT_MACHINE.to_string(), + from: card.phase, + to: target_phase, + context: Some(format!( + "conflict {} is already {}", + input.conflict_id, + conflict_phase_name(card.phase) + )), + }); + } + append_json( + "texo.conflict.resolve", + cx, + ::KIND, + &ConflictResolvedV2 { + conflict_id: input.conflict_id.clone(), + workspace_id: card.workspace_id, + resolution: input.resolution.clone(), + resolved_by: input.resolved_by, + observed_at_ms: input.observed_at_ms, + transition: transition_record( + CONFLICT_MACHINE, + &entity, + 1, + target_phase, + vec![TransitionCauseV1 { + lane: 0, + key: format!("conflict:{}", input.conflict_id), + }], + input.observed_at_ms, + ), + }, + )?; + Ok(ConflictResolveOutput { + conflict_id: input.conflict_id, + resolution: input.resolution, + already_applied: false, + receipt: Some(take_one_receipt("texo.conflict.resolve")?), + }) + }) +} +#[derive(Debug, Deserialize)] +struct ConflictsListInput {} + +#[derive(Debug, Serialize)] +struct ConflictsOutput { + open: Vec, + resolved: Vec, +} + +#[derive(Debug, Serialize)] +struct ConflictRow { + conflict_id: String, + claim_a: String, + claim_b: String, + subject_hint: String, + reason: String, + status: crate::claims::status::ConflictStatus, +} + +#[derive(Debug, Deserialize)] +struct ConflictsCommitInput { + observed_at_ms: u64, +} + +#[derive(Debug, Serialize)] +struct CommittedConflict { + conflict_id: String, + sequence: u64, + receipt: ReceiptNote, +} + +#[derive(Debug, Deserialize)] +struct ConflictResolveInput { + conflict_id: String, + resolution: String, + resolved_by: String, + observed_at_ms: u64, +} + +#[derive(Debug, Serialize)] +struct ConflictResolveOutput { + conflict_id: String, + resolution: String, + already_applied: bool, + receipt: Option, +} +fn conflicts_output(view: &WorkspaceView) -> ConflictsOutput { + let mut open = Vec::new(); + let mut resolved = Vec::new(); + for conflict in &view.conflicts { + let row = ConflictRow { + conflict_id: conflict.conflict_id.clone(), + claim_a: conflict.claim_a.clone(), + claim_b: conflict.claim_b.clone(), + subject_hint: conflict_subject(view, conflict), + reason: conflict.reason.clone(), + status: conflict_status(conflict), + }; + if conflict.phase == 1 { + open.push(row); + } else { + resolved.push(row); + } + } + open.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); + resolved.sort_by(|left, right| left.conflict_id.cmp(&right.conflict_id)); + ConflictsOutput { open, resolved } +} + +pub(super) fn conflict_status(conflict: &ConflictCard) -> crate::claims::status::ConflictStatus { + match conflict.phase { + 2 => crate::claims::status::ConflictStatus::Resolved, + 3 => crate::claims::status::ConflictStatus::Ignored, + _ => crate::claims::status::ConflictStatus::Open, + } +} +pub(super) fn conflict_subject(view: &WorkspaceView, conflict: &ConflictCard) -> String { + view.claims + .iter() + .find(|claim| claim.card.claim_id == conflict.claim_a) + .and_then(|claim| claim.card.subject_hint.clone()) + .unwrap_or_default() +} + +pub(super) fn claim_text(view: &WorkspaceView, claim_id: &str) -> String { + view.claims + .iter() + .find(|claim| claim.card.claim_id == claim_id) + .map(|claim| claim.card.text.clone()) + .unwrap_or_default() +} + +pub(super) fn newer_claim<'a>( + view: &'a WorkspaceView, + left: &str, + right: &str, +) -> Result<&'a ClaimCard, TexoError> { + let left = view + .claims + .iter() + .find(|claim| claim.card.claim_id == left) + .ok_or_else(|| TexoError::MissingEntity { + entity: entity_for_claim(left), + })?; + let right = view + .claims + .iter() + .find(|claim| claim.card.claim_id == right) + .ok_or_else(|| TexoError::MissingEntity { + entity: entity_for_claim(right), + })?; + if left.card.observed_at_ms >= right.card.observed_at_ms { + Ok(&left.card) + } else { + Ok(&right.card) + } +} diff --git a/src/ops/handlers/host.rs b/src/ops/handlers/host.rs new file mode 100644 index 0000000..25c9e6e --- /dev/null +++ b/src/ops/handlers/host.rs @@ -0,0 +1,24 @@ +use super::common::{parse_input, run_op}; +use crate::ops::env; +use serde::Deserialize; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = HOST_FINGERPRINT, + register = register_host_fingerprint, + register_item = host_fingerprint_item, + name = "texo.host.fingerprint", + effect = Inspect, + input_schema = "texo.host.fingerprint.input.v2", + output_schema = "texo.host.fingerprint.output.v2", + receipt_kind = "receipt.texo.host.fingerprint.v2" +)] +#[tracing::instrument(skip_all)] +fn host_fingerprint(input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.host.fingerprint", || { + let _input: HostFingerprintInput = parse_input("texo.host.fingerprint", input)?; + env::with(|op_env| op_env.host_interface.clone()) + }) +} +#[derive(Debug, Deserialize)] +struct HostFingerprintInput {} diff --git a/src/ops/handlers/ingest.rs b/src/ops/handlers/ingest.rs new file mode 100644 index 0000000..cd42308 --- /dev/null +++ b/src/ops/handlers/ingest.rs @@ -0,0 +1,615 @@ +use super::common::{ + append_json, assemble_current_view, elapsed_ms, op_runtime, parse_input, run_op, take_receipts, + workspace_temporal_policy, WORKSPACE_VIEW_PROJECTION, +}; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::ids::{claim_id_from_parts, ClaimId, SourceId}; +use crate::events::payloads::{ClaimRecordedV2, ClaimSupersededV2, SourceObservedV2}; +use crate::extract::hints::hints_from_line_normalized; +use crate::extract::markdown::{collect_markdown_files, MarkdownDocument}; +use crate::extract::normalize::normalize_line; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use batpak::event::EventPayload; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use syncbat::HandlerResult; + +mod supersession; + +pub(crate) use supersession::infer_supersessions; +pub(super) use supersession::settle_indexed_explicit_supersessions; + +const MAX_INLINE_SOURCE_FAILURES: usize = 256; +const MAX_SOURCE_FAILURE_DETAIL_CHARS: usize = 512; + +#[syncbat::operation( + descriptor = INGEST_RUN, + register = register_ingest_run, + register_item = ingest_run_item, + name = "texo.ingest.run", + effect = Persist, + input_schema = "texo.ingest.run.input.v2", + output_schema = "texo.ingest.run.output.v3", + receipt_kind = "receipt.texo.ingest.run.v2", + appends_events = ["evt.e001", "evt.e002", "evt.e003"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn ingest_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.ingest.run", || { + let input: IngestRunInput = parse_input("texo.ingest.run", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.ingest.run", error))?; + let (root, workspace_id, config) = env::with(|op_env| { + ( + op_env.root.clone(), + op_env.workspace_id.clone(), + op_env.config.clone(), + ) + })?; + let project_started = Instant::now(); + let mut view = assemble_current_view()?; + let project_ms = elapsed_ms(project_started); + let path = resolve_path(&root, &input.path); + let plan = plan_sources( + "texo.ingest.run", + &root, + &path, + &workspace_id, + input.observed_at_ms, + config.extractor_cmd.as_deref(), + &view, + )?; + reject_failed_ingest_plan(&root, &path, &plan, &input)?; + let execution = execute_ingest_plan( + cx, + &plan, + &config, + &mut view, + input.observed_at_ms, + input.dry_run, + project_ms, + )?; + finish_ingest_output(&root, workspace_id, plan, &input, execution) + }) +} + +fn reject_failed_ingest_plan( + root: &Path, + path: &Path, + plan: &SourcePlan, + input: &IngestRunInput, +) -> Result<(), TexoError> { + if plan.skipped.is_empty() || (!input.strict && (plan.succeeded > 0 || plan.empty)) { + return Ok(()); + } + let sample = serde_json::to_string(&plan.skipped.iter().take(8).cloned().collect::>())?; + let (_, artifact) = settle_source_failures(root, input.observed_at_ms, plan.skipped.clone())?; + Err(TexoError::Source { + path: path.to_string_lossy().to_string(), + detail: format!( + "{} source(s) failed during planning; strict={} good_sources={}; sample={sample}; artifact={}", + plan.skipped.len(), + input.strict, + plan.succeeded, + artifact.as_deref().unwrap_or("inline") + ), + }) +} + +#[derive(Debug, Deserialize)] +pub(crate) struct IngestRunInput { + pub(crate) path: PathBuf, + pub(crate) dry_run: bool, + #[serde(default)] + pub(crate) strict: bool, + pub(crate) observed_at_ms: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum IngestCompletion { + Complete, + Partial, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub(crate) enum SourceFailureCode { + #[serde(rename = "source.utf8")] + Utf8, + #[serde(rename = "source.io")] + Io, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct SourceSkipRow { + pub(crate) path: String, + pub(crate) code: SourceFailureCode, + pub(crate) detail: String, +} + +#[derive(Debug, Serialize)] +pub(crate) struct IngestRunOutput { + pub(crate) outcome: IngestCompletion, + pub(crate) workspace_id: String, + pub(crate) sources_observed: u32, + pub(crate) claims_recorded: u32, + pub(crate) claims_superseded: u32, + pub(crate) supersessions_held: usize, + pub(crate) held_supersessions: Vec, + pub(crate) dry_run: bool, + pub(crate) empty: bool, + pub(crate) skipped: Vec, + pub(crate) skipped_total: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) skipped_artifact: Option, + pub(crate) phase_ms: IngestPhaseMs, + pub(crate) events_appended: u64, + pub(crate) receipts: Vec, +} + +#[derive(Debug, Serialize)] +pub(crate) struct IngestPhaseMs { + pub(crate) discover: u64, + pub(crate) extract: u64, + pub(crate) append: u64, + pub(crate) project: u64, +} + +pub(crate) struct PlannedSource { + pub(crate) observed: SourceObservedV2, + pub(crate) claims: Vec, +} + +pub(crate) struct SourcePlan { + pub(crate) sources: Vec, + pub(crate) skipped: Vec, + pub(crate) empty: bool, + pub(crate) succeeded: usize, + pub(crate) discover_ms: u64, + pub(crate) extract_ms: u64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct CmdClaimLine { + pub(crate) line_start: u32, + pub(crate) text: String, + pub(crate) normalized_text: String, + pub(crate) subject_hint: Option, + pub(crate) predicate_hint: Option, + pub(crate) object_hint: Option, + pub(crate) confidence_ppm: u32, + pub(crate) char_start: Option, + pub(crate) char_end: Option, + pub(crate) extractor_model: Option, + pub(crate) prompt_version: Option, +} +/// Closed reason an explicit replacement proposal could not become authority. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExplicitSupersessionHoldReason { + /// The proposed successor is on an older source revision. + TemporalReversed, + /// Both source revisions are valid but incomparable. + TemporalConcurrent, + /// Available source evidence cannot establish an order. + TemporalUnknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +/// One explicit replacement proposal held by temporal policy. +pub struct HeldExplicitSupersession { + /// Claim that would have been retired. + pub old_claim_id: ClaimId, + /// Proposed successor claim. + pub new_claim_id: ClaimId, + /// Typed reason no authority-bearing event was appended. + pub reason: ExplicitSupersessionHoldReason, +} + +/// Applied and held outcomes from the shared explicit-replacement policy. +pub(crate) struct ExplicitSupersessionOutcome { + /// Authority-bearing supersession events safe to append. + pub applied: Vec, + /// Proposals withheld pending authoritative source order. + pub held: Vec, +} + +pub(super) struct IngestExecution { + source_count: u32, + claim_count: u32, + supersede_count: u32, + held_supersessions: Vec, + append_ms: u64, + project_ms: u64, +} + +pub(super) fn execute_ingest_plan( + cx: &mut syncbat::Ctx<'_>, + plan: &SourcePlan, + config: &crate::config::WorkspaceConfig, + view: &mut std::sync::Arc, + observed_at_ms: u64, + dry_run: bool, + mut project_ms: u64, +) -> Result { + if dry_run { + let source_count = u32::try_from(plan.sources.len()).unwrap_or(u32::MAX); + let claim_count = plan.sources.iter().fold(0_u32, |count, source| { + count.saturating_add(u32::try_from(source.claims.len()).unwrap_or(u32::MAX)) + }); + return Ok(IngestExecution { + source_count, + claim_count, + supersede_count: 0, + held_supersessions: Vec::new(), + append_ms: 0, + project_ms, + }); + } + let append_started = Instant::now(); + let (source_count, claim_count) = append_planned_sources(cx, &plan.sources)?; + let mut append_ms = elapsed_ms(append_started); + let project_started = Instant::now(); + *view = assemble_current_view()?; + project_ms = project_ms.saturating_add(elapsed_ms(project_started)); + let (supersede_count, held_supersessions, inference_ms) = + infer_and_append_ingest_supersessions(cx, plan, config, view, observed_at_ms)?; + append_ms = append_ms.saturating_add(inference_ms); + Ok(IngestExecution { + source_count, + claim_count, + supersede_count, + held_supersessions, + append_ms, + project_ms, + }) +} + +fn append_planned_sources( + cx: &mut syncbat::Ctx<'_>, + sources: &[PlannedSource], +) -> Result<(u32, u32), TexoError> { + let mut source_count = 0_u32; + let mut claim_count = 0_u32; + for source in sources { + append_json( + "texo.ingest.run", + cx, + ::KIND, + &source.observed, + )?; + source_count = source_count.saturating_add(1); + for claim in &source.claims { + append_json( + "texo.ingest.run", + cx, + ::KIND, + claim, + )?; + claim_count = claim_count.saturating_add(1); + } + } + Ok((source_count, claim_count)) +} + +fn infer_and_append_ingest_supersessions( + cx: &mut syncbat::Ctx<'_>, + plan: &SourcePlan, + config: &crate::config::WorkspaceConfig, + view: &WorkspaceView, + observed_at_ms: u64, +) -> Result<(u32, Vec, u64), TexoError> { + if config + .semantics + .as_ref() + .is_some_and(|semantics| semantics.enabled) + { + return Ok((0, Vec::new(), 0)); + } + let new_claims = plan + .sources + .iter() + .flat_map(|source| source.claims.iter().cloned()) + .collect::>(); + let temporal = workspace_temporal_policy(view)?; + let inference = infer_supersessions(view, &new_claims, observed_at_ms, &temporal)?; + let append_started = Instant::now(); + let mut count = 0_u32; + for superseded in inference.applied { + append_json( + "texo.ingest.run", + cx, + ::KIND, + &superseded, + )?; + count = count.saturating_add(1); + } + Ok((count, inference.held, elapsed_ms(append_started))) +} + +pub(super) fn finish_ingest_output( + root: &Path, + workspace_id: String, + plan: SourcePlan, + input: &IngestRunInput, + execution: IngestExecution, +) -> Result { + let outcome = if plan.skipped.is_empty() { + IngestCompletion::Complete + } else { + IngestCompletion::Partial + }; + let skipped_total = plan.skipped.len(); + let (skipped, skipped_artifact) = + settle_source_failures(root, input.observed_at_ms, plan.skipped)?; + let events_appended = if input.dry_run { + 0 + } else { + u64::from(execution.source_count) + .saturating_add(u64::from(execution.claim_count)) + .saturating_add(u64::from(execution.supersede_count)) + }; + Ok(IngestRunOutput { + outcome, + workspace_id, + sources_observed: execution.source_count, + claims_recorded: execution.claim_count, + claims_superseded: execution.supersede_count, + supersessions_held: execution.held_supersessions.len(), + held_supersessions: execution.held_supersessions, + dry_run: input.dry_run, + empty: plan.empty, + skipped, + skipped_total, + skipped_artifact, + phase_ms: IngestPhaseMs { + discover: plan.discover_ms, + extract: plan.extract_ms, + append: execution.append_ms, + project: execution.project_ms, + }, + events_appended, + receipts: if input.dry_run { + Vec::new() + } else { + take_receipts()? + }, + }) +} + +pub(crate) fn resolve_path(root: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + root.join(path) + } +} + +pub(crate) fn plan_sources( + op: &str, + root: &Path, + input_path: &Path, + workspace_id: &str, + observed_at_ms: u64, + extractor_cmd: Option<&str>, + view: &WorkspaceView, +) -> Result { + let existing_hashes = view + .sources + .iter() + .map(|source| source.body_hash_hex.clone()) + .collect::>(); + let mut batch_hashes = BTreeSet::new(); + let mut planned = Vec::new(); + let discover_started = Instant::now(); + let discovery = collect_markdown_files(input_path).map_err(|error| TexoError::Source { + path: input_path.to_string_lossy().to_string(), + detail: error.to_string(), + })?; + let discover_ms = elapsed_ms(discover_started); + let empty = discovery.files.is_empty(); + let mut skipped = discovery + .failures + .into_iter() + .map(|failure| SourceSkipRow { + path: failure.path.to_string_lossy().to_string(), + code: SourceFailureCode::Io, + detail: bounded_source_detail(&failure.error.to_string()), + }) + .collect::>(); + let mut succeeded = 0; + let extract_started = Instant::now(); + for path in discovery.files { + let doc = match MarkdownDocument::from_path(&path, root) { + Ok(doc) => doc, + Err(error) => { + skipped.push(SourceSkipRow { + path: path.to_string_lossy().to_string(), + code: match error { + crate::extract::markdown::SourceError::Utf8(_) => SourceFailureCode::Utf8, + crate::extract::markdown::SourceError::Io(_) + | crate::extract::markdown::SourceError::Walk(_) + | crate::extract::markdown::SourceError::Id(_) => SourceFailureCode::Io, + }, + detail: bounded_source_detail(&error.to_string()), + }); + continue; + } + }; + succeeded += 1; + if existing_hashes.contains(&doc.body_hash_hex) + || !batch_hashes.insert(doc.body_hash_hex.clone()) + { + continue; + } + let claims = if let Some(cmd) = extractor_cmd { + extract_cmd_claims(op, root, cmd, &path, &doc, workspace_id, observed_at_ms)? + } else { + extract_heuristic_claims(&doc, workspace_id, observed_at_ms)? + }; + planned.push(PlannedSource { + observed: SourceObservedV2 { + source_id: doc.source_id, + workspace_id: workspace_id.to_string(), + source_kind: "markdown".to_string(), + path: doc.path, + body_hash_hex: doc.body_hash_hex, + observed_at_ms, + }, + claims, + }); + } + skipped.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(SourcePlan { + sources: planned, + skipped, + empty, + succeeded, + discover_ms, + extract_ms: elapsed_ms(extract_started), + }) +} + +fn bounded_source_detail(detail: &str) -> String { + detail + .chars() + .take(MAX_SOURCE_FAILURE_DETAIL_CHARS) + .collect() +} + +fn settle_source_failures( + root: &Path, + observed_at_ms: u64, + rows: Vec, +) -> Result<(Vec, Option), TexoError> { + if rows.len() <= MAX_INLINE_SOURCE_FAILURES { + return Ok((rows, None)); + } + let bytes = serde_json::to_vec(&rows)?; + let digest = blake3::hash(&bytes).to_hex().to_string(); + let short_digest: String = digest.chars().take(16).collect(); + let relative = PathBuf::from(".texo") + .join("operations") + .join("ingest-skips") + .join(format!("{observed_at_ms}-{short_digest}.json")); + let path = root.join(&relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary = path.with_extension("json.tmp"); + std::fs::write(&temporary, bytes)?; + std::fs::rename(temporary, path)?; + Ok(( + rows.into_iter().take(MAX_INLINE_SOURCE_FAILURES).collect(), + Some(relative.to_string_lossy().to_string()), + )) +} + +fn extract_heuristic_claims( + doc: &MarkdownDocument, + workspace_id: &str, + observed_at_ms: u64, +) -> Result, TexoError> { + let source_id = SourceId::try_from(doc.source_id.as_str())?; + let mut claims = Vec::new(); + for line in &doc.lines { + let normalized = normalize_line(&line.text); + let Some(hints) = hints_from_line_normalized(&line.text, &normalized) else { + continue; + }; + let claim_id = claim_id_from_parts(&source_id, line.number, &normalized).to_string(); + let char_start = saturating_u32(line.char_start); + let char_end = saturating_u32(line.char_start.saturating_add(line.text.len())); + claims.push(ClaimRecordedV2 { + claim_id, + workspace_id: workspace_id.to_string(), + source_id: doc.source_id.clone(), + source_path: doc.path.clone(), + line_start: line.number, + line_end: line.number, + char_start, + char_end, + text: line.text.clone(), + normalized_text: normalized, + subject_hint: Some(hints.subject_hint), + predicate_hint: Some(hints.predicate_hint), + object_hint: Some(hints.object_hint), + confidence_ppm: hints.confidence_ppm, + extractor_kind: "heuristic-v1".to_string(), + extractor_model: String::new(), + prompt_version: String::new(), + observed_at_ms, + }); + } + Ok(claims) +} + +/// Execute the workspace-configured extractor command. +/// +/// This is an explicit local-code-execution trust boundary: anyone who can +/// write workspace configuration selects code executed by the extractor. Texo +/// stages only the selected input and runs the command through the fail-closed +/// bvisor adapter; there is no unconfined fallback. +fn extract_cmd_claims( + op: &str, + _root: &Path, + cmd: &str, + path: &Path, + doc: &MarkdownDocument, + workspace_id: &str, + observed_at_ms: u64, +) -> Result, TexoError> { + let output = + crate::compat::bvisor::run_extractor(cmd, path).map_err(|error| TexoError::Extract { + detail: format!("{op}: confined extractor failed: {error}"), + })?; + let stdout = String::from_utf8(output).map_err(|error| TexoError::Extract { + detail: format!("{op}: extractor stdout was not utf-8: {error}"), + })?; + let source_id = SourceId::try_from(doc.source_id.as_str())?; + let mut claims = Vec::new(); + for (idx, line) in stdout.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let parsed: CmdClaimLine = + serde_json::from_str(line).map_err(|error| TexoError::Extract { + detail: format!("{op}: extractor line {} json error: {error}", idx + 1), + })?; + let claim_id = + claim_id_from_parts(&source_id, parsed.line_start, &parsed.normalized_text).to_string(); + claims.push(ClaimRecordedV2 { + claim_id, + workspace_id: workspace_id.to_string(), + source_id: doc.source_id.clone(), + source_path: doc.path.clone(), + line_start: parsed.line_start, + line_end: parsed.line_start, + char_start: parsed.char_start.unwrap_or(0), + char_end: parsed.char_end.unwrap_or(0), + text: parsed.text, + normalized_text: parsed.normalized_text, + subject_hint: parsed.subject_hint, + predicate_hint: parsed.predicate_hint, + object_hint: parsed.object_hint, + confidence_ppm: parsed.confidence_ppm, + extractor_kind: "extractor-cmd".to_string(), + extractor_model: parsed.extractor_model.unwrap_or_default(), + prompt_version: parsed.prompt_version.unwrap_or_default(), + observed_at_ms, + }); + } + Ok(claims) +} + +fn saturating_u32(value: usize) -> u32 { + // Source byte offsets are journaled as v2 u32 fields; extremely large inputs + // saturate rather than truncating silently. + u32::try_from(value).unwrap_or(u32::MAX) +} diff --git a/src/ops/handlers/ingest/supersession.rs b/src/ops/handlers/ingest/supersession.rs new file mode 100644 index 0000000..2252905 --- /dev/null +++ b/src/ops/handlers/ingest/supersession.rs @@ -0,0 +1,196 @@ +use super::super::common::{append_json, take_receipts, workspace_temporal_policy_through}; +use super::{ + ExplicitSupersessionHoldReason, ExplicitSupersessionOutcome, HeldExplicitSupersession, +}; +use crate::claims::workspace::{ClaimView, WorkspaceView}; +use crate::error::TexoError; +use crate::events::coordinate::entity_for_claim; +use crate::events::ids::ClaimId; +use crate::events::machines::{transition_record, TransitionCauseV1, CLAIM_MACHINE}; +use crate::events::payloads::{ClaimRecordedV2, ClaimSupersededV2}; +use crate::knowledge::TemporalRelation; +use crate::ops::env::ReceiptNote; +use crate::semantics::pipeline::RelateTemporalPolicy; +use batpak::event::EventPayload; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone)] +struct ExplicitReplacementCandidate { + claim_id: ClaimId, + workspace_id: String, + source_id: String, + normalized_text: String, + subject_hint: Option, +} + +pub(crate) fn infer_supersessions( + view: &WorkspaceView, + new_claims: &[ClaimRecordedV2], + observed_at_ms: u64, + temporal: &RelateTemporalPolicy, +) -> Result { + let candidates = new_claims + .iter() + .map(|claim| { + Ok(ExplicitReplacementCandidate { + claim_id: ClaimId::try_from(claim.claim_id.as_str())?, + workspace_id: claim.workspace_id.clone(), + source_id: claim.source_id.clone(), + normalized_text: claim.normalized_text.clone(), + subject_hint: claim.subject_hint.clone(), + }) + }) + .collect::, TexoError>>()?; + Ok(infer_explicit_supersessions( + view, + &candidates, + observed_at_ms, + temporal, + )) +} + +fn infer_indexed_supersessions( + view: &WorkspaceView, + new_claim_ids: &BTreeSet, + observed_at_ms: u64, + temporal: &RelateTemporalPolicy, +) -> ExplicitSupersessionOutcome { + let candidates = view + .claims + .iter() + .filter_map(|claim| { + let claim_id = ClaimId::try_from(claim.card.claim_id.as_str()).ok()?; + new_claim_ids + .contains(&claim_id) + .then(|| ExplicitReplacementCandidate { + claim_id, + workspace_id: claim.card.workspace_id.clone(), + source_id: claim.card.source_id.clone(), + normalized_text: claim.card.normalized_text.clone(), + subject_hint: claim.card.subject_hint.clone(), + }) + }) + .collect::>(); + infer_explicit_supersessions(view, &candidates, observed_at_ms, temporal) +} + +pub(in crate::ops::handlers) fn settle_indexed_explicit_supersessions( + cx: &mut syncbat::Ctx<'_>, + view: &WorkspaceView, + new_claim_ids: &BTreeSet, + observed_at_ms: u64, + receipts: &mut Vec, +) -> Result { + let evidence_frontier = receipts + .iter() + .map(|receipt| receipt.global_sequence) + .max() + .unwrap_or(view.frontier); + let temporal = workspace_temporal_policy_through(view, evidence_frontier)?; + let outcome = infer_indexed_supersessions(view, new_claim_ids, observed_at_ms, &temporal); + for supersession in &outcome.applied { + append_json( + "texo.knowledge.index", + cx, + ::KIND, + supersession, + )?; + } + receipts.extend(take_receipts()?); + Ok(outcome) +} + +fn infer_explicit_supersessions( + view: &WorkspaceView, + new_claims: &[ExplicitReplacementCandidate], + observed_at_ms: u64, + temporal: &RelateTemporalPolicy, +) -> ExplicitSupersessionOutcome { + let mut by_subject: BTreeMap, Vec<&ClaimView>> = BTreeMap::new(); + for claim in &view.claims { + by_subject + .entry(claim.card.subject_hint.clone()) + .or_default() + .push(claim); + } + let mut applied = Vec::new(); + let mut held = Vec::new(); + let mut seen_old = BTreeSet::new(); + for new_claim in new_claims { + if !replacement_signal(&new_claim.normalized_text) { + continue; + } + let Some(candidates) = by_subject.get(&new_claim.subject_hint) else { + continue; + }; + for old in candidates { + if old.card.claim_id == new_claim.claim_id.as_str() + || old.card.phase != 1 + || old.card.normalized_text == new_claim.normalized_text + { + continue; + } + let Ok(old_claim_id) = ClaimId::try_from(old.card.claim_id.as_str()) else { + continue; + }; + let hold_reason = match temporal.compare_claims(&old_claim_id, &new_claim.claim_id) { + None | Some(TemporalRelation::Same | TemporalRelation::Before) => None, + Some(TemporalRelation::After) => { + Some(ExplicitSupersessionHoldReason::TemporalReversed) + } + Some(TemporalRelation::Concurrent) => { + Some(ExplicitSupersessionHoldReason::TemporalConcurrent) + } + Some(TemporalRelation::Unknown) => { + Some(ExplicitSupersessionHoldReason::TemporalUnknown) + } + }; + if !seen_old.insert(old.card.claim_id.clone()) { + continue; + } + if let Some(reason) = hold_reason { + held.push(HeldExplicitSupersession { + old_claim_id, + new_claim_id: new_claim.claim_id.clone(), + reason, + }); + continue; + } + let old_entity = entity_for_claim(&old.card.claim_id); + applied.push(ClaimSupersededV2 { + old_claim_id: old.card.claim_id.clone(), + new_claim_id: new_claim.claim_id.to_string(), + workspace_id: new_claim.workspace_id.clone(), + reason: "explicit replacement wording accepted by temporal policy".to_string(), + decided_by: "texo.ingest.run".to_string(), + observed_at_ms, + transition: transition_record( + CLAIM_MACHINE, + &old_entity, + 1, + 2, + vec![TransitionCauseV1 { + lane: 0, + key: format!("ingest:{}", new_claim.source_id), + }], + observed_at_ms, + ), + }); + } + } + applied.sort_by(|left, right| { + left.old_claim_id + .cmp(&right.old_claim_id) + .then_with(|| left.new_claim_id.cmp(&right.new_claim_id)) + }); + held.sort_by(|left, right| { + left.old_claim_id + .cmp(&right.old_claim_id) + .then_with(|| left.new_claim_id.cmp(&right.new_claim_id)) + }); + ExplicitSupersessionOutcome { applied, held } +} + +fn replacement_signal(normalized_text: &str) -> bool { + crate::lexicon::contains_replacement_signal(normalized_text) +} diff --git a/src/ops/handlers/knowledge.rs b/src/ops/handlers/knowledge.rs new file mode 100644 index 0000000..a45f9f9 --- /dev/null +++ b/src/ops/handlers/knowledge.rs @@ -0,0 +1,558 @@ +use super::common::{ + append_json, assemble_current_view, assemble_snapshot_view, evidence_projection_through, + latest_source_snapshot, op_runtime, parse_input, plan_and_attach_snapshot_relations, + recorded_capture_limits, run_op, source_snapshot_by_id, source_snapshots_through, + take_one_receipt, take_receipts, WORKSPACE_VIEW_PROJECTION, +}; +use super::ingest::{settle_indexed_explicit_supersessions, HeldExplicitSupersession}; +use super::knowledge_read::{ + append_knowledge_plan, latest_code_index, load_code_artifact_at, plan_claim_evidence, + triangulate_from_view, +}; +use super::model::AgentClaimRow; +use crate::code_index::{ + build as build_code_index, load as load_code_index, persist as persist_code_index, read_scip, + CodeIndexLimits, +}; +use crate::error::{SnapshotFailureKind, TexoError}; +use crate::events::ids::WorkspaceId; +use crate::events::payloads::{CodeIndexRecordedV1, SourceSnapshotRecordedV1}; +use crate::git_source::{capture as capture_git, CaptureLimits, GitCapture}; +use crate::knowledge::{ + AnalysisQuality, AnswerState, ClaimEvidence, CodeIndexId, CodeOccurrence, CoverageGap, + CoverageGapKind, KnowledgeCoverage, RepositoryId, SnapshotRead, TriangulationTarget, + UncertaintyReason, +}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use crate::ops::reconcile::append_proposals as append_reconciliation_proposals; +use crate::reconcile::{ + claims_from_view as reconcile_claims, evaluate_with_backends, plan_candidates, + unresolved_row as reconcile_unresolved_row, KnowledgeReconcileInput, KnowledgeReconcileOutput, + ReconcileBackendOutput, ReconcileCompletion, +}; +use batpak::event::EventPayload; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = KNOWLEDGE_TRIANGULATE, + register = register_knowledge_triangulate, + register_item = knowledge_triangulate_item, + name = "texo.knowledge.triangulate", + effect = Inspect, + input_schema = "texo.knowledge.triangulate.input.v1", + output_schema = "texo.knowledge.triangulate.output.v1", + receipt_kind = "receipt.texo.knowledge.triangulate.v1", + reads_events = ["evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e"], + queries_projections = ["texo.workspace.view.v2", "texo.evidence.view.v1"] +)] +#[tracing::instrument(skip_all)] +fn knowledge_triangulate(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.knowledge.triangulate", || { + let input: KnowledgeTriangulateInput = parse_input("texo.knowledge.triangulate", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.knowledge.triangulate", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + triangulate_from_view(&view, &snapshot, input.target) + }) +} +#[syncbat::operation( + descriptor = KNOWLEDGE_INDEX, + register = register_knowledge_index, + register_item = knowledge_index_item, + name = "texo.knowledge.index", + effect = Persist, + input_schema = "texo.knowledge.index.input.v1", + output_schema = "texo.knowledge.index.output.v2", + receipt_kind = "receipt.texo.knowledge.index.v1", + appends_events = ["evt.e003", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00f"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn knowledge_index(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.knowledge.index", || { + let input: KnowledgeIndexInput = parse_input("texo.knowledge.index", input)?; + let limits = input.validated_limits()?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.knowledge.index", error))?; + let view = assemble_current_view()?; + let (root, workspace_id) = + env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; + let workspace = WorkspaceId::new(workspace_id.clone())?; + let previous_snapshots = source_snapshots_through(Some(view.frontier))?; + let repository_id = repository_id_for_index(&root, &workspace_id, &previous_snapshots); + let mut capture = capture_git(&root, repository_id, limits)?; + let already_indexed = latest_source_snapshot(Some(view.frontier))? + .is_some_and(|existing| existing.snapshot_id == capture.snapshot_id); + + let planned = plan_claim_evidence( + &view, + &capture.sources, + &capture.snapshot_id, + input.observed_at_ms, + )?; + for gap in planned.gaps { + if capture.coverage.gaps.len() < 256 { + capture.coverage.gaps.push(gap); + } else { + capture.coverage.truncated = true; + } + } + if !planned.rows.is_empty() { + capture.coverage.analysis_quality = AnalysisQuality::Syntactic; + } + capture.coverage.occurrences = u64::try_from(planned.rows.len()).unwrap_or(u64::MAX); + let relations = plan_and_attach_snapshot_relations( + &root, + &workspace, + &mut capture, + &previous_snapshots, + input.observed_at_ms, + )?; + let snapshot = SourceSnapshotRecordedV1 { + workspace_id: workspace.clone(), + repository_id: capture.repository_id, + snapshot_id: capture.snapshot_id.clone(), + base_commit: capture.base_commit.clone(), + base_tree: capture.base_tree, + index_digest_hex: capture.index_digest_hex, + overlay_digest_hex: capture.overlay_digest_hex, + dirty: capture.dirty, + coverage: capture.coverage.clone(), + capture_max_files: u64::try_from(limits.max_files).unwrap_or(u64::MAX), + capture_max_file_bytes: limits.max_file_bytes, + capture_max_total_bytes: limits.max_total_bytes, + observed_at_ms: input.observed_at_ms, + }; + let indexed_claim_ids = planned + .rows + .iter() + .map(|(_, link)| link.claim_id.clone()) + .collect::>(); + let mut receipts = append_knowledge_plan( + cx, + &workspace, + &snapshot, + &planned.rows, + &relations, + input.observed_at_ms, + )?; + let supersessions = settle_indexed_explicit_supersessions( + cx, + &view, + &indexed_claim_ids, + input.observed_at_ms, + &mut receipts, + )?; + Ok(KnowledgeIndexOutput { + workspace_id, + snapshot_id: capture.snapshot_id, + base_commit: capture.base_commit, + dirty: capture.dirty, + sources_captured: capture.sources.len(), + evidence_recorded: planned.rows.len(), + claims_linked: planned.rows.len(), + relations_recorded: relations.len(), + supersessions_applied: supersessions.applied.len(), + supersessions_held: supersessions.held.len(), + held_supersessions: supersessions.held, + already_indexed, + coverage: capture.coverage, + receipts, + }) + }) +} +#[syncbat::operation( + descriptor = CODE_INDEX_BUILD, + register = register_code_index_build, + register_item = code_index_build_item, + name = "texo.code.index.build", + effect = Persist, + input_schema = "texo.code.index.build.input.v1", + output_schema = "texo.code.index.build.output.v2", + receipt_kind = "receipt.texo.code.index.build.v1", + appends_events = ["evt.e00e"], + reads_events = ["evt.e00b", "evt.e00e"], + queries_projections = ["texo.code.index.v1"] +)] +#[tracing::instrument(skip_all)] +fn code_index_build(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.code.index.build", || { + let input: CodeIndexBuildInput = parse_input("texo.code.index.build", input)?; + let limits = input.validated_limits()?; + let (root, workspace_id) = + env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; + let (recorded, capture) = recorded_source_capture(&root, &input)?; + if input.can_reuse_default() { + if let Some(existing) = latest_code_index(None, &recorded.snapshot_id)? { + if let Some(artifact) = + load_code_index(&root, &existing.index_id, &existing.artifact_digest_hex)? + { + if artifact.snapshot_id != recorded.snapshot_id { + return Err(TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: "code-index artifact belongs to a different source snapshot" + .to_string(), + }); + } + return Ok(CodeIndexBuildOutput { + workspace_id, + snapshot_id: existing.snapshot_id, + index_id: existing.index_id.clone(), + format: artifact.format, + analyzer_fingerprint: artifact.analyzer_fingerprint, + artifact_digest_hex: existing.artifact_digest_hex, + artifact_path: format!( + ".texo/cache/code-index/{}.bin", + existing.index_id.as_str() + ), + coverage: artifact.coverage, + already_indexed: true, + receipt: None, + }); + } + } + } + let scip_bytes = input + .scip_path + .as_deref() + .map(|path| read_scip(&root, path, limits.max_scip_bytes)) + .transpose()?; + let prepared = build_code_index(&capture, scip_bytes.as_deref(), limits)?; + let artifact_path = persist_code_index(&root, &prepared)?; + let payload = CodeIndexRecordedV1 { + workspace_id: WorkspaceId::new(workspace_id.clone())?, + snapshot_id: recorded.snapshot_id, + index_id: prepared.artifact.index_id.clone(), + format: prepared.artifact.format, + analyzer_fingerprint: prepared.artifact.analyzer_fingerprint.clone(), + artifact_digest_hex: prepared.artifact_digest_hex.clone(), + coverage: prepared.artifact.coverage.clone(), + observed_at_ms: input.observed_at_ms, + }; + append_json( + "texo.code.index.build", + cx, + ::KIND, + &payload, + )?; + let relative_artifact = artifact_path + .strip_prefix(&root) + .unwrap_or(&artifact_path) + .to_string_lossy() + .to_string(); + Ok(CodeIndexBuildOutput { + workspace_id, + snapshot_id: payload.snapshot_id, + index_id: payload.index_id, + format: payload.format, + analyzer_fingerprint: payload.analyzer_fingerprint, + artifact_digest_hex: payload.artifact_digest_hex, + artifact_path: relative_artifact, + coverage: payload.coverage, + already_indexed: false, + receipt: Some(take_one_receipt("texo.code.index.build")?), + }) + }) +} +#[syncbat::operation( + descriptor = KNOWLEDGE_RECONCILE, + register = register_knowledge_reconcile, + register_item = knowledge_reconcile_item, + name = "texo.knowledge.reconcile", + effect = Persist, + input_schema = "texo.knowledge.reconcile.input.v1", + output_schema = "texo.knowledge.reconcile.output.v1", + receipt_kind = "receipt.texo.knowledge.reconcile.v1", + appends_events = ["evt.e00c", "evt.e00d", "evt.e010"], + reads_events = ["evt.e002", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e"], + queries_projections = ["texo.workspace.view.v2", "texo.code.index.v1"], + requires_capabilities = ["texo.cap.model"] +)] +#[tracing::instrument(skip_all)] +fn knowledge_reconcile(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.knowledge.reconcile", || { + let input: KnowledgeReconcileInput = parse_input("texo.knowledge.reconcile", input)?; + let (limits, budget_secs, concurrency) = input.validated()?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.knowledge.reconcile", error))?; + let view = assemble_current_view()?; + let snapshot = + latest_source_snapshot(Some(view.frontier))?.ok_or_else(|| TexoError::OpInput { + op: "texo.knowledge.reconcile".to_string(), + detail: "no source snapshot exists; run `texo index` first".to_string(), + })?; + let loaded = load_code_artifact_at(view.frontier, &snapshot.snapshot_id)?; + let artifact = loaded.artifact.ok_or_else(|| TexoError::OpInput { + op: "texo.knowledge.reconcile".to_string(), + detail: "the current source snapshot has no available code index; run `texo index`" + .to_string(), + })?; + let claims = reconcile_claims(&view)?; + let plan = plan_candidates(&claims, &artifact, limits); + let candidates_considered = plan.candidates.len(); + let evidence = evidence_projection_through(view.frontier)?; + let mut already_linked = 0; + let candidates = plan + .candidates + .into_iter() + .filter(|candidate| { + let linked = evidence + .for_claim(candidate.claim_id.as_str()) + .iter() + .any(|item| { + item.occurrence.occurrence_id == candidate.occurrence.occurrence_id + }); + already_linked += usize::from(linked); + !linked + }) + .collect::>(); + let (root, gateway) = + env::with(|op_env| (op_env.root.clone(), op_env.config.gateway.clone()))?; + let backend = evaluate_with_backends( + &root, + gateway.as_ref(), + &candidates, + std::time::Duration::from_secs(budget_secs), + concurrency, + )?; + let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; + let ReconcileBackendOutput { + proposals, + unresolved: backend_unresolved, + judge_fingerprint, + } = backend; + let (accepted, rejected) = append_reconciliation_proposals( + cx, + &workspace_id, + input.observed_at_ms, + limits.min_score_ppm, + &judge_fingerprint, + proposals, + )?; + let unresolved = backend_unresolved + .iter() + .map(reconcile_unresolved_row) + .collect::>(); + let mut coverage = artifact.coverage; + if plan.truncated { + coverage.truncated = true; + if !coverage + .gaps + .iter() + .any(|gap| gap.kind == CoverageGapKind::BudgetExceeded) + { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::BudgetExceeded, + }); + } + } + let partial = coverage.truncated || !coverage.gaps.is_empty() || !unresolved.is_empty(); + Ok(KnowledgeReconcileOutput { + outcome: if partial { + ReconcileCompletion::Partial + } else { + ReconcileCompletion::Complete + }, + snapshot_id: snapshot.snapshot_id, + candidates_considered, + already_linked, + accepted, + rejected, + unresolved, + coverage, + receipts: take_receipts()?, + }) + }) +} + +fn repository_id_for_index( + root: &Path, + workspace_id: &str, + previous: &[SourceSnapshotRecordedV1], +) -> RepositoryId { + previous.last().cloned().map_or_else( + || { + let canonical = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + RepositoryId::derive(&format!( + "texo.repository.v1\u{1f}{workspace_id}\u{1f}{}", + canonical.display() + )) + }, + |snapshot| snapshot.repository_id, + ) +} + +fn recorded_source_capture( + root: &Path, + input: &CodeIndexBuildInput, +) -> Result<(SourceSnapshotRecordedV1, GitCapture), TexoError> { + let recorded = match input.snapshot_id.as_ref() { + Some(wanted) => source_snapshot_by_id(wanted)?.ok_or_else(|| TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: "requested source snapshot is not recorded; run `texo index` first".to_string(), + })?, + None => latest_source_snapshot(None)?.ok_or_else(|| TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: "run `texo index` to record a Git source snapshot first".to_string(), + })?, + }; + let limits = recorded_capture_limits(&recorded); + let capture = capture_git(root, recorded.repository_id.clone(), limits)?; + if capture.snapshot_id != recorded.snapshot_id { + return Err(TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: + "Git commit/index/worktree changed; run source indexing again before code indexing" + .to_string(), + }); + } + Ok((recorded, capture)) +} +#[derive(Debug, Deserialize)] +struct KnowledgeIndexInput { + observed_at_ms: u64, + #[serde(default)] + max_files: Option, + #[serde(default)] + max_file_bytes: Option, + #[serde(default)] + max_total_bytes: Option, +} + +impl KnowledgeIndexInput { + fn validated_limits(&self) -> Result { + let defaults = CaptureLimits::default(); + let limits = CaptureLimits { + max_files: self.max_files.unwrap_or(defaults.max_files), + max_file_bytes: self.max_file_bytes.unwrap_or(defaults.max_file_bytes), + max_total_bytes: self.max_total_bytes.unwrap_or(defaults.max_total_bytes), + }; + if limits.max_files == 0 + || limits.max_files > 100_000 + || limits.max_file_bytes == 0 + || limits.max_file_bytes > 16 * 1024 * 1024 + || limits.max_total_bytes == 0 + || limits.max_total_bytes > 512 * 1024 * 1024 + { + return Err(TexoError::OpInput { + op: "texo.knowledge.index".to_string(), + detail: "capture limits must be non-zero and at most 100000 files, 16 MiB per file, and 512 MiB total".to_string(), + }); + } + Ok(limits) + } +} + +#[derive(Debug, Serialize)] +struct KnowledgeIndexOutput { + workspace_id: String, + snapshot_id: crate::knowledge::SourceSnapshotId, + base_commit: crate::knowledge::GitObjectId, + dirty: bool, + sources_captured: usize, + evidence_recorded: usize, + claims_linked: usize, + relations_recorded: usize, + supersessions_applied: usize, + supersessions_held: usize, + held_supersessions: Vec, + already_indexed: bool, + coverage: KnowledgeCoverage, + receipts: Vec, +} + +#[derive(Debug, Deserialize)] +struct CodeIndexBuildInput { + #[serde(default)] + snapshot_id: Option, + #[serde(default)] + scip_path: Option, + observed_at_ms: u64, + #[serde(default)] + max_scip_bytes: Option, + #[serde(default)] + max_documents: Option, + #[serde(default)] + max_occurrences: Option, + #[serde(default)] + analysis_budget_secs: Option, +} + +impl CodeIndexBuildInput { + fn can_reuse_default(&self) -> bool { + self.scip_path.is_none() + && self.max_scip_bytes.is_none() + && self.max_documents.is_none() + && self.max_occurrences.is_none() + && self.analysis_budget_secs.is_none() + } + + fn validated_limits(&self) -> Result { + let defaults = CodeIndexLimits::default(); + let limits = CodeIndexLimits { + max_scip_bytes: self.max_scip_bytes.unwrap_or(defaults.max_scip_bytes), + max_documents: self.max_documents.unwrap_or(defaults.max_documents), + max_occurrences: self.max_occurrences.unwrap_or(defaults.max_occurrences), + analysis_budget: std::time::Duration::from_secs( + self.analysis_budget_secs + .unwrap_or(defaults.analysis_budget.as_secs()), + ), + }; + if limits.max_scip_bytes == 0 + || limits.max_scip_bytes > 256 * 1024 * 1024 + || limits.max_documents == 0 + || limits.max_documents > 100_000 + || limits.max_occurrences == 0 + || limits.max_occurrences > 2_000_000 + || limits.analysis_budget.is_zero() + || limits.analysis_budget > std::time::Duration::from_secs(300) + { + return Err(TexoError::OpInput { + op: "texo.code.index.build".to_string(), + detail: "code-index limits must be non-zero and at most 256 MiB, 100000 documents, 2000000 occurrences, and 300 seconds".to_string(), + }); + } + Ok(limits) + } +} + +#[derive(Debug, Serialize)] +struct CodeIndexBuildOutput { + workspace_id: String, + snapshot_id: crate::knowledge::SourceSnapshotId, + index_id: CodeIndexId, + format: crate::knowledge::CodeIndexFormat, + analyzer_fingerprint: String, + artifact_digest_hex: String, + artifact_path: String, + coverage: KnowledgeCoverage, + already_indexed: bool, + receipt: Option, +} +#[derive(Debug, Deserialize)] +struct KnowledgeTriangulateInput { + target: TriangulationTarget, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct KnowledgeTriangulateOutput { + pub(super) target: TriangulationTarget, + pub(super) answer_state: AnswerState, + pub(super) assertions: Vec, + pub(super) evidence: Vec, + pub(super) structural_evidence: Vec, + pub(super) uncertainty: Vec, + pub(super) coverage: KnowledgeCoverage, + pub(super) settlement_complete: bool, + pub(super) snapshot: SnapshotRead, +} diff --git a/src/ops/handlers/knowledge_read.rs b/src/ops/handlers/knowledge_read.rs new file mode 100644 index 0000000..760c4cc --- /dev/null +++ b/src/ops/handlers/knowledge_read.rs @@ -0,0 +1,632 @@ +use super::claims::claim_list_rows; +use super::common::{append_json, coverage_for_view, evidence_projection_through, take_receipts}; +use super::knowledge::KnowledgeTriangulateOutput; +use super::model::AgentClaimRow; +use super::relate::settlement_is_complete; +use crate::claims::workspace::WorkspaceView; +use crate::code_index::load as load_code_index; +use crate::error::{SnapshotFailureKind, TexoError}; +use crate::events::coordinate::{entity_for_claim, scope_for_workspace}; +use crate::events::ids::{ClaimId, WorkspaceId}; +use crate::events::payloads::{ + ClaimEvidenceLinkedV1, CodeIndexRecordedV1, EvidenceOccurrenceRecordedV1, + SourceSnapshotRecordedV1, SourceSnapshotRelationV1, +}; +use crate::git_source::{CapturedLayer, CapturedSource}; +use crate::knowledge::{ + AnalysisQuality, AnswerState, ByteRange, ClaimEvidence, CodeIndexArtifact, CodeOccurrence, + CoverageGap, CoverageGapKind, EvidenceLinkMethod, EvidenceOccurrence, EvidenceOccurrenceId, + EvidenceSourceKind, EvidenceStance, KnowledgeCoverage, LineRange, SnapshotRead, + TriangulationTarget, UncertaintyReason, MAX_EVIDENCE_EXCERPT_BYTES, +}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use batpak::coordinate::Region; +use batpak::event::EventPayload; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +const MAX_TRIANGULATION_CODE_OCCURRENCES: usize = 200; + +pub(super) fn triangulate_from_view( + view: &WorkspaceView, + snapshot: &SnapshotRead, + target: TriangulationTarget, +) -> Result { + validate_triangulation_target(&target)?; + let projection = evidence_projection_through(view.frontier)?; + let claim_ids = triangulation_claim_ids(view, &target)?; + let assertions = claim_list_rows(view, None)? + .into_iter() + .filter(|claim| claim_ids.contains(&claim.claim_id)) + .collect::>(); + let mut evidence = claim_ids + .iter() + .flat_map(|claim_id| projection.for_claim(claim_id).iter().cloned()) + .collect::>(); + evidence.retain(|item| evidence_matches_target(item, &target)); + let mut coverage = coverage_for_view(view, snapshot)?; + let code = code_evidence_for_target(view.frontier, snapshot, &target)?; + if let Some(code_coverage) = &code.coverage { + merge_coverage(&mut coverage, code_coverage); + } + if projection.is_incomplete() { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::AnalysisIncomplete, + }); + } + let settlement_complete = settlement_is_complete(view)?; + let mut uncertainty = BTreeSet::new(); + if snapshot.descriptor.source_snapshot_id.is_none() { + uncertainty.insert(UncertaintyReason::SourceSnapshotUnavailable); + } + if coverage.truncated || !coverage.gaps.is_empty() { + uncertainty.insert(UncertaintyReason::PartialCoverage); + } + if !settlement_complete { + uncertainty.insert(UncertaintyReason::SettlementIncomplete); + } + if matches!(target, TriangulationTarget::Symbol { .. }) && code.unavailable { + uncertainty.insert(UncertaintyReason::CodeIndexUnavailable); + if !coverage + .gaps + .iter() + .any(|gap| gap.kind == CoverageGapKind::CodeIndexUnavailable) + { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::CodeIndexUnavailable, + }); + } + } + if !assertions.is_empty() && evidence.is_empty() { + uncertainty.insert(UncertaintyReason::ExactEvidenceUnavailable); + } + let answer_state = answer_state_for_rows(&assertions, &evidence, &code.rows); + Ok(KnowledgeTriangulateOutput { + target, + answer_state, + assertions, + evidence, + structural_evidence: code.rows, + uncertainty: uncertainty.into_iter().collect(), + coverage, + settlement_complete, + snapshot: snapshot.clone(), + }) +} + +#[derive(Default)] +pub(super) struct CodeEvidenceLookup { + rows: Vec, + coverage: Option, + unavailable: bool, +} + +#[derive(Default)] +pub(super) struct LoadedCodeArtifact { + pub(super) artifact: Option, + pub(super) coverage: Option, + pub(super) unavailable: bool, +} + +pub(super) fn code_evidence_for_target( + frontier: u64, + snapshot: &SnapshotRead, + target: &TriangulationTarget, +) -> Result { + let Some(source_snapshot_id) = snapshot.descriptor.source_snapshot_id.as_ref() else { + return Ok(CodeEvidenceLookup { + unavailable: true, + ..CodeEvidenceLookup::default() + }); + }; + let loaded = load_code_artifact_at(frontier, source_snapshot_id)?; + let Some(artifact) = loaded.artifact else { + return Ok(CodeEvidenceLookup { + coverage: loaded.coverage, + unavailable: loaded.unavailable, + ..CodeEvidenceLookup::default() + }); + }; + let mut rows = artifact + .occurrences + .into_iter() + .filter(|occurrence| code_occurrence_matches(occurrence, target)) + .take(MAX_TRIANGULATION_CODE_OCCURRENCES + 1) + .collect::>(); + let mut coverage = artifact.coverage; + if rows.len() > MAX_TRIANGULATION_CODE_OCCURRENCES { + rows.truncate(MAX_TRIANGULATION_CODE_OCCURRENCES); + coverage.truncated = true; + if !coverage + .gaps + .iter() + .any(|gap| gap.path.is_none() && gap.kind == CoverageGapKind::BudgetExceeded) + { + coverage.gaps.push(CoverageGap { + path: None, + kind: CoverageGapKind::BudgetExceeded, + }); + } + } + Ok(CodeEvidenceLookup { + rows, + coverage: Some(coverage), + unavailable: false, + }) +} + +pub(super) fn load_code_artifact_at( + frontier: u64, + source_snapshot_id: &crate::knowledge::SourceSnapshotId, +) -> Result { + let Some(recorded) = latest_code_index(Some(frontier), source_snapshot_id)? else { + return Ok(LoadedCodeArtifact { + unavailable: true, + ..LoadedCodeArtifact::default() + }); + }; + let artifact = env::with(|op_env| { + load_code_index( + &op_env.root, + &recorded.index_id, + &recorded.artifact_digest_hex, + ) + })??; + if artifact + .as_ref() + .is_some_and(|artifact| artifact.snapshot_id != *source_snapshot_id) + { + return Err(TexoError::Snapshot { + kind: SnapshotFailureKind::SourceUnavailable, + detail: "code-index artifact belongs to a different source snapshot".to_string(), + }); + } + Ok(LoadedCodeArtifact { + unavailable: artifact.is_none(), + coverage: Some(recorded.coverage), + artifact, + }) +} + +pub(super) fn latest_code_index( + frontier: Option, + snapshot_id: &crate::knowledge::SourceSnapshotId, +) -> Result, TexoError> { + env::with(|op_env| { + let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); + let mut after = None; + let mut latest = None; + 'pages: loop { + let page = op_env.store.query_entries_after(®ion, after, 256); + if page.is_empty() { + break; + } + for entry in &page { + if frontier.is_some_and(|frontier| entry.global_sequence() > frontier) { + break 'pages; + } + if entry.event_kind() == ::KIND { + let raw = op_env.store.read_raw(entry.event_id())?; + let payload = + batpak::encoding::from_bytes::(&raw.event.payload) + .map_err(|error| TexoError::Decode { + entity: entry.coord().entity().to_string(), + detail: error.to_string(), + })?; + if payload.snapshot_id == *snapshot_id { + latest = Some(payload); + } + } + } + after = page.last().map(batpak::store::IndexEntry::global_sequence); + } + Ok::<_, TexoError>(latest) + })? +} + +pub(super) fn code_occurrence_matches( + occurrence: &CodeOccurrence, + target: &TriangulationTarget, +) -> bool { + match target { + TriangulationTarget::Claim { .. } => false, + TriangulationTarget::Path { + path, + line_start, + line_end, + } => { + occurrence.path == *path + && line_start.is_none_or(|start| occurrence.line_range.end >= start) + && line_end.is_none_or(|end| occurrence.line_range.start <= end) + } + TriangulationTarget::Symbol { symbol } => { + occurrence.symbol == *symbol || occurrence.display_name == *symbol + } + } +} + +pub(super) fn merge_coverage(target: &mut KnowledgeCoverage, code: &KnowledgeCoverage) { + if analysis_quality_rank(code.analysis_quality) > analysis_quality_rank(target.analysis_quality) + { + target.analysis_quality = code.analysis_quality; + } + target.sources_examined = target.sources_examined.max(code.sources_examined); + target.occurrences = target.occurrences.saturating_add(code.occurrences); + target.truncated |= code.truncated; + for gap in &code.gaps { + if target.gaps.len() >= 256 { + target.truncated = true; + break; + } + if !target.gaps.contains(gap) { + target.gaps.push(gap.clone()); + } + } +} + +const fn analysis_quality_rank(quality: AnalysisQuality) -> u8 { + match quality { + AnalysisQuality::Precise => 3, + AnalysisQuality::Syntactic => 2, + AnalysisQuality::Lexical => 1, + AnalysisQuality::Unavailable => 0, + } +} + +pub(super) fn validate_triangulation_target(target: &TriangulationTarget) -> Result<(), TexoError> { + match target { + TriangulationTarget::Claim { claim_id } if claim_id.is_empty() => Err(TexoError::OpInput { + op: "texo.knowledge.triangulate".to_string(), + detail: "claim_id must not be empty".to_string(), + }), + TriangulationTarget::Path { + path, + line_start, + line_end, + } => { + let safe = !path.is_empty() + && !Path::new(path).is_absolute() + && Path::new(path) + .components() + .all(|component| matches!(component, std::path::Component::Normal(_))); + let valid_range = match (*line_start, *line_end) { + (None, None) => true, + (Some(start), Some(end)) => start > 0 && start <= end, + _ => false, + }; + if safe && valid_range { + Ok(()) + } else { + Err(TexoError::OpInput { + op: "texo.knowledge.triangulate".to_string(), + detail: "path must be repository-relative and line bounds must be absent or an ordered one-based pair".to_string(), + }) + } + } + TriangulationTarget::Symbol { symbol } if symbol.is_empty() || symbol.len() > 1024 => { + Err(TexoError::OpInput { + op: "texo.knowledge.triangulate".to_string(), + detail: "symbol must contain between 1 and 1024 bytes".to_string(), + }) + } + TriangulationTarget::Claim { .. } | TriangulationTarget::Symbol { .. } => Ok(()), + } +} + +pub(super) fn triangulation_claim_ids( + view: &WorkspaceView, + target: &TriangulationTarget, +) -> Result, TexoError> { + match target { + TriangulationTarget::Claim { claim_id } => { + if view + .claims + .iter() + .any(|claim| claim.card.claim_id == *claim_id) + { + Ok(BTreeSet::from([claim_id.clone()])) + } else { + Err(TexoError::MissingEntity { + entity: entity_for_claim(claim_id), + }) + } + } + TriangulationTarget::Path { + path, + line_start, + line_end, + } => Ok(view + .claims + .iter() + .filter(|claim| claim.card.source_path == *path) + .filter(|claim| { + line_start.is_none_or(|start| claim.card.line_end >= start) + && line_end.is_none_or(|end| claim.card.line_start <= end) + }) + .map(|claim| claim.card.claim_id.clone()) + .collect()), + TriangulationTarget::Symbol { .. } => Ok(BTreeSet::new()), + } +} + +pub(super) fn evidence_matches_target( + evidence: &ClaimEvidence, + target: &TriangulationTarget, +) -> bool { + match target { + TriangulationTarget::Claim { .. } => true, + TriangulationTarget::Path { + path, + line_start, + line_end, + } => { + evidence.occurrence.path == *path + && line_start.is_none_or(|start| evidence.occurrence.line_range.end >= start) + && line_end.is_none_or(|end| evidence.occurrence.line_range.start <= end) + } + TriangulationTarget::Symbol { .. } => false, + } +} + +pub(super) fn answer_state_for_rows( + assertions: &[AgentClaimRow], + evidence: &[ClaimEvidence], + structural_evidence: &[CodeOccurrence], +) -> AnswerState { + use crate::claims::status::ClaimStatus; + if evidence + .iter() + .any(|item| item.stance == EvidenceStance::Contradicts) + { + AnswerState::Contradicted + } else if assertions + .iter() + .any(|claim| claim.status == ClaimStatus::Conflicting) + { + AnswerState::Incomparable + } else if assertions + .iter() + .any(|claim| claim.status == ClaimStatus::Superseded) + { + AnswerState::Stale + } else if (!assertions.is_empty() + && evidence + .iter() + .any(|item| item.stance == EvidenceStance::Supports)) + || !structural_evidence.is_empty() + { + AnswerState::Supported + } else { + AnswerState::Unverified + } +} + +pub(super) fn answer_state_for_claim( + status: Option, + evidence: &[ClaimEvidence], +) -> AnswerState { + use crate::claims::status::ClaimStatus; + match status { + Some(ClaimStatus::Superseded) => AnswerState::Stale, + Some(ClaimStatus::Conflicting) => AnswerState::Incomparable, + Some(ClaimStatus::Current) + if evidence + .iter() + .any(|item| item.stance == EvidenceStance::Contradicts) => + { + AnswerState::Contradicted + } + Some(ClaimStatus::Current) + if evidence + .iter() + .any(|item| item.stance == EvidenceStance::Supports) => + { + AnswerState::Supported + } + Some(ClaimStatus::Current) | None => AnswerState::Unverified, + } +} + +pub(super) struct EvidencePlan { + pub(super) rows: Vec<(EvidenceOccurrence, ClaimEvidenceLinkedV1)>, + pub(super) gaps: Vec, +} + +pub(super) fn append_knowledge_plan( + cx: &mut syncbat::Ctx<'_>, + workspace_id: &WorkspaceId, + snapshot: &SourceSnapshotRecordedV1, + rows: &[(EvidenceOccurrence, ClaimEvidenceLinkedV1)], + relations: &[SourceSnapshotRelationV1], + observed_at_ms: u64, +) -> Result, TexoError> { + append_json( + "texo.knowledge.index", + cx, + ::KIND, + snapshot, + )?; + for (occurrence, link) in rows { + append_json( + "texo.knowledge.index", + cx, + ::KIND, + &EvidenceOccurrenceRecordedV1 { + workspace_id: workspace_id.clone(), + occurrence: occurrence.clone(), + observed_at_ms, + }, + )?; + append_json( + "texo.knowledge.index", + cx, + ::KIND, + link, + )?; + } + for relation in relations { + append_json( + "texo.knowledge.index", + cx, + ::KIND, + relation, + )?; + } + take_receipts() +} + +pub(super) fn plan_claim_evidence( + view: &WorkspaceView, + sources: &[CapturedSource], + snapshot_id: &crate::knowledge::SourceSnapshotId, + observed_at_ms: u64, +) -> Result { + let by_path = sources + .iter() + .map(|source| (source.path.as_str(), source)) + .collect::>(); + let workspace_id = WorkspaceId::new(view.workspace_id.clone())?; + let mut rows = Vec::new(); + let mut gaps = Vec::new(); + for claim in &view.claims { + match plan_claim_evidence_row( + claim, + by_path.get(claim.card.source_path.as_str()).copied(), + snapshot_id, + &workspace_id, + observed_at_ms, + )? { + PlannedEvidence::Skip => {} + PlannedEvidence::Gap(gap) => gaps.push(gap), + PlannedEvidence::Row(row) => rows.push(*row), + } + } + Ok(EvidencePlan { rows, gaps }) +} + +enum PlannedEvidence { + Skip, + Gap(CoverageGap), + Row(Box<(EvidenceOccurrence, ClaimEvidenceLinkedV1)>), +} + +fn plan_claim_evidence_row( + claim: &crate::claims::workspace::ClaimView, + source: Option<&CapturedSource>, + snapshot_id: &crate::knowledge::SourceSnapshotId, + workspace_id: &WorkspaceId, + observed_at_ms: u64, +) -> Result { + let Some(source) = source else { + return Ok(PlannedEvidence::Skip); + }; + let source_digest_hex = crate::events::ids::blake3_bytes_hex(&source.bytes); + let captured_source_id = crate::events::ids::source_id_from_hash(&source_digest_hex)?; + if captured_source_id.as_str() != claim.card.source_id { + return Ok(PlannedEvidence::Gap(CoverageGap { + path: Some(source.path.clone()), + kind: CoverageGapKind::AnalysisIncomplete, + })); + } + let Some((start, end)) = + line_byte_range(&source.bytes, claim.card.line_start, claim.card.line_end) + else { + return Ok(PlannedEvidence::Gap(CoverageGap { + path: Some(source.path.clone()), + kind: CoverageGapKind::AnalysisIncomplete, + })); + }; + let excerpt_bytes = &source.bytes[start..end]; + let Ok(excerpt) = std::str::from_utf8(excerpt_bytes) else { + return Ok(PlannedEvidence::Gap(CoverageGap { + path: Some(source.path.clone()), + kind: CoverageGapKind::UnsupportedEncoding, + })); + }; + if excerpt.len() > MAX_EVIDENCE_EXCERPT_BYTES { + return Ok(PlannedEvidence::Gap(CoverageGap { + path: Some(source.path.clone()), + kind: CoverageGapKind::SourceTooLarge, + })); + } + let material = format!( + "texo.evidence.occurrence.v1\u{1f}{snapshot_id}\u{1f}{}\u{1f}{start}\u{1f}{end}\u{1f}{}", + source.path, claim.card.claim_id + ); + let occurrence_id = EvidenceOccurrenceId::derive(&material); + let occurrence = EvidenceOccurrence { + occurrence_id: occurrence_id.clone(), + snapshot_id: snapshot_id.clone(), + source_kind: match source.layer { + CapturedLayer::Committed => EvidenceSourceKind::GitBlob, + CapturedLayer::Worktree => EvidenceSourceKind::WorktreeOverlay, + }, + path: source.path.clone(), + byte_range: ByteRange::new( + u64::try_from(start).unwrap_or(u64::MAX), + u64::try_from(end).unwrap_or(u64::MAX), + ) + .map_err(|error| TexoError::Source { + path: source.path.clone(), + detail: error.to_string(), + })?, + line_range: LineRange::new(claim.card.line_start, claim.card.line_end).map_err( + |error| TexoError::Source { + path: source.path.clone(), + detail: error.to_string(), + }, + )?, + git_blob: source.blob_id.clone(), + source_digest_hex, + excerpt: excerpt.to_string(), + analyzer_fingerprint: format!( + "{}:{}:{}", + claim.card.extractor_kind, claim.card.extractor_model, claim.card.prompt_version + ), + analysis_quality: AnalysisQuality::Syntactic, + }; + occurrence.validate().map_err(|error| TexoError::Source { + path: source.path.clone(), + detail: error.to_string(), + })?; + let link = ClaimEvidenceLinkedV1 { + workspace_id: workspace_id.clone(), + claim_id: ClaimId::try_from(claim.card.claim_id.as_str())?, + occurrence_id, + stance: EvidenceStance::Supports, + method: EvidenceLinkMethod::Deterministic, + observed_at_ms, + }; + Ok(PlannedEvidence::Row(Box::new((occurrence, link)))) +} + +pub(super) fn line_byte_range( + bytes: &[u8], + start_line: u32, + end_line: u32, +) -> Option<(usize, usize)> { + if start_line == 0 || end_line < start_line { + return None; + } + let mut line = 1_u32; + let mut line_start = 0_usize; + let mut range_start = None; + for offset in 0..=bytes.len() { + let boundary = offset == bytes.len() || bytes.get(offset) == Some(&b'\n'); + if !boundary { + continue; + } + if line == start_line { + range_start = Some(line_start); + } + if line == end_line { + return range_start.map(|start| (start, offset)); + } + line = line.saturating_add(1); + line_start = offset.saturating_add(1); + } + None +} diff --git a/src/ops/handlers/model.rs b/src/ops/handlers/model.rs new file mode 100644 index 0000000..e935e6a --- /dev/null +++ b/src/ops/handlers/model.rs @@ -0,0 +1,26 @@ +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub(crate) struct AgentClaimRow { + pub(crate) claim_id: String, + pub(crate) status: crate::claims::status::ClaimStatus, + pub(crate) subject_hint: Option, + pub(crate) text: String, + pub(crate) source: AgentSourceRow, + pub(crate) receipt: AgentReceiptRow, + pub(crate) supersedes: Vec, + pub(crate) superseded_by: Option, +} + +#[derive(Debug, Serialize)] +pub(crate) struct AgentSourceRow { + pub(crate) source_id: String, + pub(crate) path: String, + pub(crate) line_start: u32, +} + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct AgentReceiptRow { + pub(crate) event_id: String, + pub(crate) sequence: u64, +} diff --git a/src/ops/handlers/relate.rs b/src/ops/handlers/relate.rs new file mode 100644 index 0000000..266a4b0 --- /dev/null +++ b/src/ops/handlers/relate.rs @@ -0,0 +1,664 @@ +pub(super) use self::authority::authoritative_settlements; +use self::authority::{rejudge_authoritative_pair, RejudgePairContext}; +use self::backend::{relate_with_backends, semantic_claims_from_view, RelateBackendContext}; +use self::campaign::{ + append_checkpoint, candidate_policy_digest, latest_checkpoint, resolve_start, CampaignBasis, + CampaignStart, CheckpointDraft, +}; +pub(super) use self::campaign::{require_complete_settlement, settlement_is_complete}; +use self::publication::{ + append_relate_conflicts, append_relate_supersessions, append_relation_deferrals, + append_relation_judgments, relate_publication, +}; +use super::common::{ + assemble_current_view, op_runtime, parse_input, run_op, semantic_temporal_policy, + take_receipts, WORKSPACE_VIEW_PROJECTION, +}; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::ids::{ClaimId, WorkspaceId}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use crate::relate::settlement::CampaignPhase; +use crate::semantics::pipeline::{ + CandidateCursor, ClaimView as SemanticClaimView, RelateOutcome, RelateTemporalPolicy, + RelateThresholds, DEFAULT_CANDIDATE_PAIR_BUDGET, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::path::PathBuf; +use std::sync::Arc; +use syncbat::HandlerResult; + +const RELATE_PREFILTER: f32 = 0.60; +#[cfg(feature = "openrouter")] +pub(super) const ENV_RELATE_CACHE: &str = "TEXO_RELATE_CACHE"; +#[cfg(feature = "openrouter")] +pub(super) const DEFAULT_RELATE_CACHE: &str = ".texo/relate-cache"; + +mod authority; +mod backend; +mod campaign; +mod publication; +#[syncbat::operation( + descriptor = RELATE_RUN, + register = register_relate_run, + register_item = relate_run_item, + name = "texo.relate.run", + effect = Persist, + input_schema = "texo.relate.run.input.v2", + output_schema = "texo.relate.run.output.v2", + receipt_kind = "receipt.texo.relate.run.v2", + appends_events = ["evt.e003", "evt.e004", "evt.e009", "evt.e00a", "evt.e012"], + queries_projections = ["texo.workspace.view.v2"], + requires_capabilities = ["texo.cap.model"] +)] +#[tracing::instrument(skip_all)] +fn relate_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.relate.run", || { + let input: RelateRunInput = parse_input("texo.relate.run", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.relate.run", error))?; + run_relate_pass( + "texo.relate.run", + cx, + input.observed_at_ms, + input.validated_options()?, + ) + }) +} + +#[derive(Debug, Deserialize)] +struct RelateRunInput { + observed_at_ms: u64, + #[serde(default)] + strict: bool, + #[serde(default)] + max_candidate_pairs: Option, + #[serde(default)] + candidate_cursor: Option, + #[serde(default)] + rejudge_pair: Option>, +} + +impl RelateRunInput { + fn validated_options(&self) -> Result { + let candidate_pair_budget = self + .max_candidate_pairs + .unwrap_or(DEFAULT_CANDIDATE_PAIR_BUDGET); + if candidate_pair_budget == 0 { + return Err(TexoError::OpInput { + op: "texo.relate.run".to_string(), + detail: "max_candidate_pairs must be greater than zero".to_string(), + }); + } + let rejudge_pair = match self.rejudge_pair.as_deref() { + None => None, + Some([older, newer]) => Some(( + ClaimId::try_from(older.as_str())?, + ClaimId::try_from(newer.as_str())?, + )), + Some(_) => { + return Err(TexoError::OpInput { + op: "texo.relate.run".to_string(), + detail: "rejudge_pair must contain exactly two claim ids".to_string(), + }); + } + }; + Ok(RelatePassOptions { + strict: self.strict, + candidate_pair_budget, + candidate_cursor: self.candidate_cursor.map(CandidateCursor::from_offset), + rejudge_pair, + }) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct RelatePassOptions { + strict: bool, + candidate_pair_budget: usize, + candidate_cursor: Option, + rejudge_pair: Option<(ClaimId, ClaimId)>, +} + +impl RelatePassOptions { + pub(crate) fn best_effort() -> Self { + Self { + strict: false, + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + candidate_cursor: None, + rejudge_pair: None, + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum RelateCompletion { + Complete, + Partial, +} + +#[derive(Debug, Serialize)] +pub(crate) struct RelateRunOutput { + outcome: RelateCompletion, + pub(crate) claims_related: usize, + pub(crate) supersessions: Vec, + pub(crate) conflicts: Vec, + unresolved: Vec, + held: Vec, + warnings: Vec, + authority_warnings: Vec, + candidate_pairs: usize, + candidate_pair_budget: usize, + next_candidate_cursor: Option, + rejudged_pair: Option, + pub(crate) receipts: Vec, +} + +#[derive(Debug, Serialize)] +pub(crate) struct RelateSupersessionRow { + old_claim_id: String, + new_claim_id: String, + reason: String, + cache_key: String, +} + +#[derive(Debug, Serialize)] +pub(crate) struct RelateConflictRow { + conflict_id: String, + claim_a: String, + claim_b: String, + reason: String, + cache_key: String, +} + +#[derive(Debug, Serialize)] +struct RejudgedPairRow { + older_claim: String, + newer_claim: String, + prior_relation: crate::semantics::ClaimRelation, + fresh_relation: crate::semantics::ClaimRelation, + score_ppm: u32, + judge_fingerprint: String, + cache_key: String, +} + +pub(crate) fn run_relate_pass( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, + options: RelatePassOptions, +) -> Result { + let prepared = prepare_relate_pass(options)?; + match prepared.execution { + CampaignExecution::NoopComplete => { + let mut output = + empty_relate_output(prepared.claims.len(), prepared.candidate_pair_budget); + output.receipts = take_receipts()?; + return Ok(output); + } + CampaignExecution::RejudgeComplete => { + return rejudge_completed_campaign(op, cx, observed_at_ms, prepared); + } + CampaignExecution::Run(_) => {} + } + if prepared.rejudge_pair.is_some() && prepared.claims.len() < 2 { + return Err(TexoError::OpInput { + op: op.to_string(), + detail: "rejudge pair is no longer present in the current semantic claim set" + .to_string(), + }); + } + if prepared.claims.len() < 2 { + return complete_trivial_campaign(op, cx, observed_at_ms, prepared); + } + let evaluated = evaluate_relate_pass(op, cx, observed_at_ms, &prepared)?; + publish_relate_pass(op, cx, observed_at_ms, prepared, evaluated) +} + +struct PreparedRelatePass { + strict: bool, + candidate_pair_budget: usize, + candidate_cursor: CandidateCursor, + rejudge_pair: Option<(ClaimId, ClaimId)>, + view: Arc, + claims: Vec<(ClaimId, SemanticClaimView)>, + settings: RelateSettings, + evaluated_basis: CampaignBasis, + evaluated_basis_digest: String, + policy_digest: String, + execution: CampaignExecution, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CampaignExecution { + NoopComplete, + RejudgeComplete, + Run(CandidateCursor), +} + +fn campaign_execution(start: CampaignStart, rejudge_requested: bool) -> CampaignExecution { + match (start, rejudge_requested) { + (CampaignStart::AlreadyComplete, false) => CampaignExecution::NoopComplete, + (CampaignStart::AlreadyComplete, true) => CampaignExecution::RejudgeComplete, + (CampaignStart::Run(cursor), _) => CampaignExecution::Run(cursor), + } +} + +fn prepare_relate_pass(options: RelatePassOptions) -> Result { + let RelatePassOptions { + strict, + candidate_pair_budget, + candidate_cursor, + rejudge_pair, + } = options; + let view = assemble_current_view()?; + let claims = semantic_claims_from_view(&view)?; + let settings = relate_settings(&view)?; + let evaluated_basis = CampaignBasis::from_view(&view, &settings.temporal); + let evaluated_basis_digest = evaluated_basis.digest(); + let policy_digest = candidate_policy_digest( + settings.cluster, + settings.prefilter, + settings.gateway.as_ref(), + ); + let campaign_start = resolve_start( + latest_checkpoint(&view)?.as_ref(), + &evaluated_basis_digest, + &policy_digest, + candidate_cursor, + )?; + let execution = campaign_execution(campaign_start, rejudge_pair.is_some()); + let candidate_cursor = match execution { + CampaignExecution::Run(cursor) => cursor, + CampaignExecution::NoopComplete | CampaignExecution::RejudgeComplete => { + CandidateCursor::start() + } + }; + Ok(PreparedRelatePass { + strict, + candidate_pair_budget, + candidate_cursor, + rejudge_pair, + view, + claims, + settings, + evaluated_basis, + evaluated_basis_digest, + policy_digest, + execution, + }) +} + +fn rejudge_completed_campaign( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, + prepared: PreparedRelatePass, +) -> Result { + let requested = prepared + .rejudge_pair + .as_ref() + .ok_or_else(|| TexoError::OpInput { + op: op.to_string(), + detail: "completed-campaign rejudge requires a claim pair".to_string(), + })?; + let authority = authoritative_settlements(None)?; + let rejudged_pair = rejudge_authoritative_pair( + cx, + RejudgePairContext { + op, + root: &prepared.settings.root, + gateway: prepared.settings.gateway.as_ref(), + workspace_id: &prepared.settings.workspace_id, + claims: &prepared.claims, + authority: &authority, + requested, + observed_at_ms, + }, + )?; + let authority = authoritative_settlements(None)?; + append_checkpoint( + op, + cx, + CheckpointDraft { + workspace_id: prepared.settings.workspace_id, + evaluated_basis_digest_hex: prepared.evaluated_basis_digest.clone(), + result_basis_digest_hex: prepared.evaluated_basis_digest, + candidate_policy_digest_hex: prepared.policy_digest, + phase: CampaignPhase::Complete, + observed_at_ms, + }, + )?; + let mut output = empty_relate_output(prepared.claims.len(), prepared.candidate_pair_budget); + output.authority_warnings = authority.warnings; + output.rejudged_pair = Some(rejudged_pair); + output.receipts = take_receipts()?; + Ok(output) +} + +fn complete_trivial_campaign( + op: &str, + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, + prepared: PreparedRelatePass, +) -> Result { + append_checkpoint( + op, + cx, + CheckpointDraft { + workspace_id: prepared.settings.workspace_id, + evaluated_basis_digest_hex: prepared.evaluated_basis_digest.clone(), + result_basis_digest_hex: prepared.evaluated_basis_digest, + candidate_policy_digest_hex: prepared.policy_digest, + phase: CampaignPhase::Complete, + observed_at_ms, + }, + )?; + let mut output = empty_relate_output(prepared.claims.len(), prepared.candidate_pair_budget); + output.receipts = take_receipts()?; + Ok(output) +} + +struct EvaluatedRelatePass { + related: backend::SemanticRelateOutput, + authority_warnings: Vec, + rejudged_pair: Option, +} + +fn evaluate_relate_pass( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, + prepared: &PreparedRelatePass, +) -> Result { + let mut authority = authoritative_settlements(None)?; + let rejudged_pair = prepared + .rejudge_pair + .as_ref() + .map(|pair| { + rejudge_authoritative_pair( + cx, + RejudgePairContext { + op, + root: &prepared.settings.root, + gateway: prepared.settings.gateway.as_ref(), + workspace_id: &prepared.settings.workspace_id, + claims: &prepared.claims, + authority: &authority, + requested: pair, + observed_at_ms, + }, + ) + }) + .transpose()?; + if rejudged_pair.is_some() { + authority = authoritative_settlements(None)?; + } + let mut related = relate_with_backends(RelateBackendContext { + root: &prepared.settings.root, + gateway: prepared.settings.gateway.as_ref(), + claims: &prepared.claims, + thresholds: RelateThresholds { + cluster: prepared.settings.cluster, + prefilter: prepared.settings.prefilter, + }, + settled: &authority.verdicts, + temporal: &prepared.settings.temporal, + budget: std::time::Duration::from_secs(prepared.settings.budget_secs), + candidate_pair_budget: prepared.candidate_pair_budget, + candidate_cursor: prepared.candidate_cursor, + })?; + for (pair, cache_key) in &authority.cache_keys { + related.cache_keys.insert(pair.clone(), cache_key.clone()); + } + Ok(EvaluatedRelatePass { + related, + authority_warnings: authority.warnings, + rejudged_pair, + }) +} + +fn publish_relate_pass( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + observed_at_ms: u64, + prepared: PreparedRelatePass, + evaluated: EvaluatedRelatePass, +) -> Result { + let related = evaluated.related; + append_relation_judgments( + op, + cx, + &prepared.settings.workspace_id, + &related, + observed_at_ms, + )?; + append_relation_deferrals( + op, + cx, + &prepared.settings.workspace_id, + related.outcome.unresolved(), + observed_at_ms, + )?; + let publication = relate_publication(&related.outcome); + let existing_conflicts = prepared + .view + .conflicts + .iter() + .map(|conflict| conflict.conflict_id.clone()) + .collect::>(); + let supersessions = append_relate_supersessions( + op, + cx, + &prepared.view.workspace_id, + &publication.supersessions, + &related.cache_keys, + observed_at_ms, + )?; + let conflicts = append_relate_conflicts( + op, + cx, + &prepared.view.workspace_id, + &publication.conflicts, + &existing_conflicts, + &related.cache_keys, + observed_at_ms, + )?; + let (phase, result_basis_digest) = match &related.outcome { + RelateOutcome::Complete(_) => ( + CampaignPhase::Complete, + prepared + .evaluated_basis + .after_publication(&supersessions, &conflicts) + .digest(), + ), + RelateOutcome::Partial(partial) => ( + CampaignPhase::Partial { + next_candidate_cursor: partial.next_candidate_cursor.offset(), + }, + prepared.evaluated_basis_digest.clone(), + ), + }; + append_checkpoint( + op, + cx, + CheckpointDraft { + workspace_id: prepared.settings.workspace_id, + evaluated_basis_digest_hex: prepared.evaluated_basis_digest, + result_basis_digest_hex: result_basis_digest, + candidate_policy_digest_hex: prepared.policy_digest, + phase, + observed_at_ms, + }, + )?; + + finish_relate_output(RelatePublicationOutput { + claims_related: prepared.claims.len(), + outcome: related.outcome, + authority_warnings: evaluated.authority_warnings, + supersessions, + conflicts, + rejudged_pair: evaluated.rejudged_pair, + strict: prepared.strict, + }) +} + +struct RelateSettings { + root: PathBuf, + cluster: f32, + prefilter: f32, + gateway: Option, + temporal: RelateTemporalPolicy, + budget_secs: u64, + workspace_id: WorkspaceId, +} + +fn relate_settings(view: &WorkspaceView) -> Result { + let (root, cluster, prefilter, gateway) = env::with(|op_env| { + let semantics = op_env.config.semantics.as_ref(); + let cluster = semantics.map_or_else( + || crate::config::SemanticsConfig::default().cosine_threshold, + |config| config.cosine_threshold, + ); + let prefilter = semantics + .and_then(|config| config.relate_prefilter) + .unwrap_or(RELATE_PREFILTER); + ( + op_env.root.clone(), + cluster, + prefilter, + op_env.config.gateway.clone(), + ) + })?; + let budget_secs = std::env::var("TEXO_RELATE_BUDGET_SECS") + .ok() + .and_then(|value| value.parse::().ok()) + .or_else(|| gateway.as_ref().map(|config| config.relate_budget_secs)) + .unwrap_or(900); + Ok(RelateSettings { + root, + cluster, + prefilter, + gateway, + temporal: semantic_temporal_policy(view)?, + budget_secs, + workspace_id: WorkspaceId::try_from(view.workspace_id.as_str())?, + }) +} + +fn empty_relate_output(claims_related: usize, candidate_pair_budget: usize) -> RelateRunOutput { + RelateRunOutput { + outcome: RelateCompletion::Complete, + claims_related, + supersessions: Vec::new(), + conflicts: Vec::new(), + unresolved: Vec::new(), + held: Vec::new(), + warnings: Vec::new(), + authority_warnings: Vec::new(), + candidate_pairs: 0, + candidate_pair_budget, + next_candidate_cursor: None, + rejudged_pair: None, + receipts: Vec::new(), + } +} + +struct RelatePublicationOutput { + claims_related: usize, + outcome: RelateOutcome, + authority_warnings: Vec, + supersessions: Vec, + conflicts: Vec, + rejudged_pair: Option, + strict: bool, +} + +fn finish_relate_output(output: RelatePublicationOutput) -> Result { + let RelatePublicationOutput { + claims_related, + outcome, + authority_warnings, + supersessions, + conflicts, + rejudged_pair, + strict, + } = output; + let (completion, unresolved, held, candidate_pairs, candidate_pair_budget, next_cursor) = + match outcome { + RelateOutcome::Complete(complete) => ( + RelateCompletion::Complete, + Vec::new(), + complete.held, + complete.candidate_pairs, + complete.candidate_pair_budget, + None, + ), + RelateOutcome::Partial(partial) => ( + RelateCompletion::Partial, + partial.unresolved, + partial.held, + partial.candidate_pairs, + partial.candidate_pair_budget, + Some(partial.next_candidate_cursor.offset()), + ), + }; + let mut warnings = Vec::new(); + if !unresolved.is_empty() { + warnings.push( + "semantic settlement is incomplete; unresolved pairs remain authoritative gaps" + .to_string(), + ); + } + if let Some(cursor) = next_cursor { + warnings.push(format!( + "candidate page is incomplete; resume deterministically from candidate cursor {cursor}" + )); + if strict { + warnings.push( + "strict settlement withheld all derived authority until completion".to_string(), + ); + } + } + Ok(RelateRunOutput { + outcome: completion, + claims_related, + supersessions, + conflicts, + unresolved, + held, + warnings, + authority_warnings, + candidate_pairs, + candidate_pair_budget, + next_candidate_cursor: next_cursor, + rejudged_pair, + receipts: take_receipts()?, + }) +} + +#[cfg(test)] +mod tests { + use super::{campaign_execution, CampaignExecution, CampaignStart}; + use crate::semantics::pipeline::CandidateCursor; + + #[test] + fn completed_campaign_rejudge_does_not_restart_candidate_paging() { + assert_eq!( + campaign_execution(CampaignStart::AlreadyComplete, true), + CampaignExecution::RejudgeComplete + ); + assert_eq!( + campaign_execution(CampaignStart::AlreadyComplete, false), + CampaignExecution::NoopComplete + ); + assert_eq!( + campaign_execution(CampaignStart::Run(CandidateCursor::from_offset(17)), true), + CampaignExecution::Run(CandidateCursor::from_offset(17)) + ); + } +} diff --git a/src/ops/handlers/relate/authority.rs b/src/ops/handlers/relate/authority.rs new file mode 100644 index 0000000..161eef2 --- /dev/null +++ b/src/ops/handlers/relate/authority.rs @@ -0,0 +1,271 @@ +use super::super::common::append_json; +use super::backend::semantic_error; +use super::{RejudgedPairRow, DEFAULT_RELATE_CACHE, ENV_RELATE_CACHE}; +use crate::error::TexoError; +use crate::events::coordinate::scope_for_workspace; +use crate::events::ids::{ClaimId, WorkspaceId}; +use crate::events::payloads::RelationJudgedV1; +use crate::ops::env; +use crate::semantics::pipeline::ClaimView as SemanticClaimView; +use crate::semantics::score::{ppm_to_unit_interval, unit_interval_to_ppm}; +use batpak::coordinate::Region; +use batpak::event::EventPayload; +use batpak::event::EventSourced; +use batpak::store::Freshness; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +pub(in crate::ops::handlers) struct SettlementAuthority { + pub(in crate::ops::handlers) verdicts: + BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + pub(in crate::ops::handlers) cache_keys: BTreeMap<(String, String), String>, + pub(in crate::ops::handlers) warnings: Vec, + pub(in crate::ops::handlers) unresolved_pairs: usize, +} + +pub(in crate::ops::handlers) fn authoritative_settlements( + frontier: Option, +) -> Result { + env::with(|op_env| { + let entities = settlement_entities(&op_env.store, &op_env.workspace_id, frontier); + + let mut settled = BTreeMap::new(); + let mut cache_keys = BTreeMap::new(); + let mut warnings = Vec::new(); + let mut unresolved_pairs = 0; + for entity in entities { + let card = if let Some(frontier) = frontier { + let mut card = crate::claims::settlement::SettlementCard::default(); + for entry in op_env.store.by_entity(&entity) { + if entry.global_sequence() > frontier { + break; + } + let raw = op_env.store.read_raw(entry.event_id())?; + card.apply_event(&raw.event); + } + card + } else { + let Some(card) = env::deterministic_projection(|| { + op_env + .store + .project::( + &entity, + &Freshness::Consistent, + ) + })? + else { + continue; + }; + card + }; + let Some(authoritative) = card.authoritative.as_ref() else { + if !card.deferrals.is_empty() { + unresolved_pairs += 1; + } + continue; + }; + let older = ClaimId::try_from(card.older_claim.as_str())?; + let newer = ClaimId::try_from(card.newer_claim.as_str())?; + for later in &card.later_judgments { + if later.relation != authoritative.relation { + warnings.push(crate::relate::settlement::AuthorityWarning { + old_claim: older.clone(), + new_claim: newer.clone(), + prior_verdict: authoritative.relation, + prior_fingerprint: authoritative.judge_fingerprint.clone(), + new_verdict: later.relation, + new_fingerprint: later.judge_fingerprint.clone(), + message: "authoritative verdict unchanged".to_string(), + }); + } + } + settled.insert( + (older.clone(), newer.clone()), + crate::semantics::RelationVerdict { + relation: authoritative.relation.into(), + score: ppm_to_unit_interval(authoritative.score_ppm), + }, + ); + cache_keys.insert( + (older.to_string(), newer.to_string()), + authoritative.cache_key_hex.clone(), + ); + } + Ok::<_, TexoError>(SettlementAuthority { + verdicts: settled, + cache_keys, + warnings, + unresolved_pairs, + }) + })? +} + +fn settlement_entities( + store: &crate::journal_store::JournalStore, + workspace_id: &str, + frontier: Option, +) -> BTreeSet { + let region = Region::scope(scope_for_workspace(workspace_id)); + let mut after = None; + let mut entities = BTreeSet::new(); + loop { + let page = store.query_entries_after(®ion, after, 256); + if page.is_empty() { + break; + } + for entry in &page { + if frontier.is_some_and(|limit| entry.global_sequence() > limit) { + break; + } + if entry.coord().entity().starts_with("relation:") { + entities.insert(entry.coord().entity().to_string()); + } + } + if page + .last() + .is_some_and(|entry| frontier.is_some_and(|limit| entry.global_sequence() > limit)) + { + break; + } + after = page.last().map(batpak::store::IndexEntry::global_sequence); + } + entities +} + +#[derive(Clone, Copy)] +pub(super) struct RejudgePairContext<'a> { + pub(super) op: &'static str, + pub(super) root: &'a Path, + pub(super) gateway: Option<&'a crate::gateway::GatewayConfig>, + pub(super) workspace_id: &'a WorkspaceId, + pub(super) claims: &'a [(ClaimId, SemanticClaimView)], + pub(super) authority: &'a SettlementAuthority, + pub(super) requested: &'a (ClaimId, ClaimId), + pub(super) observed_at_ms: u64, +} + +pub(super) fn rejudge_authoritative_pair( + cx: &mut syncbat::Ctx<'_>, + context: RejudgePairContext<'_>, +) -> Result { + let RejudgePairContext { + op, + root, + gateway, + workspace_id, + claims, + authority, + requested, + observed_at_ms, + } = context; + let (older, newer, prior) = authority + .verdicts + .get(requested) + .map(|verdict| (requested.0.clone(), requested.1.clone(), *verdict)) + .or_else(|| { + let reversed = (requested.1.clone(), requested.0.clone()); + authority + .verdicts + .get(&reversed) + .map(|verdict| (reversed.0, reversed.1, *verdict)) + }) + .ok_or_else(|| TexoError::OpInput { + op: op.to_string(), + detail: format!( + "rejudge pair {} {} has no journal-authoritative judgment", + requested.0, requested.1 + ), + })?; + let by_id = claims + .iter() + .map(|(id, view)| (id.as_str(), view)) + .collect::>(); + let old_view = by_id + .get(older.as_str()) + .ok_or_else(|| TexoError::OpInput { + op: op.to_string(), + detail: format!("rejudge older claim {older} is not current semantic input"), + })?; + let new_view = by_id + .get(newer.as_str()) + .ok_or_else(|| TexoError::OpInput { + op: op.to_string(), + detail: format!("rejudge newer claim {newer} is not current semantic input"), + })?; + let fresh = fresh_pair_judgment(root, gateway, &old_view.text, &new_view.text)?; + append_json( + op, + cx, + ::KIND, + &RelationJudgedV1 { + workspace_id: workspace_id.clone(), + older_claim: older.clone(), + newer_claim: newer.clone(), + relation: fresh.verdict.relation.into(), + score_ppm: unit_interval_to_ppm(fresh.verdict.score), + judge_fingerprint: fresh.judge_fingerprint.clone(), + cache_key_hex: fresh.cache_key.clone(), + observed_at_ms, + }, + )?; + Ok(RejudgedPairRow { + older_claim: older.to_string(), + newer_claim: newer.to_string(), + prior_relation: prior.relation, + fresh_relation: fresh.verdict.relation, + score_ppm: unit_interval_to_ppm(fresh.verdict.score), + judge_fingerprint: fresh.judge_fingerprint, + cache_key: fresh.cache_key, + }) +} + +struct FreshPairJudgment { + verdict: crate::semantics::RelationVerdict, + judge_fingerprint: String, + cache_key: String, +} + +#[cfg(feature = "openrouter")] +fn fresh_pair_judgment( + root: &Path, + gateway: Option<&crate::gateway::GatewayConfig>, + older: &str, + newer: &str, +) -> Result { + use crate::extract::cache::CachingRelater; + use crate::semantics::openrouter::OpenRouterRelater; + use crate::semantics::ClaimRelater as _; + + let cache_dir = std::env::var_os(ENV_RELATE_CACHE) + .map_or_else(|| root.join(DEFAULT_RELATE_CACHE), PathBuf::from); + let relater = CachingRelater::new( + OpenRouterRelater::new(None, gateway).map_err(semantic_error)?, + cache_dir, + ); + let cache_key = relater.cache_key(older, newer); + relater + .evict(older, newer) + .map_err(|error| TexoError::Semantics { + backend: "relate-cache".to_string(), + detail: format!("cannot evict {cache_key}: {error}"), + })?; + let verdict = relater.relate(older, newer).map_err(semantic_error)?; + Ok(FreshPairJudgment { + verdict, + judge_fingerprint: relater.fingerprint(), + cache_key, + }) +} + +#[cfg(not(feature = "openrouter"))] +fn fresh_pair_judgment( + _root: &Path, + _gateway: Option<&crate::gateway::GatewayConfig>, + _older: &str, + _newer: &str, +) -> Result { + Err(TexoError::Semantics { + backend: "openrouter".to_string(), + detail: "openrouter feature is disabled".to_string(), + }) +} diff --git a/src/ops/handlers/relate/backend.rs b/src/ops/handlers/relate/backend.rs new file mode 100644 index 0000000..4141580 --- /dev/null +++ b/src/ops/handlers/relate/backend.rs @@ -0,0 +1,204 @@ +use super::super::common::claim_record_receipts; +use super::{DEFAULT_RELATE_CACHE, ENV_RELATE_CACHE}; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::coordinate::{entity_for_claim, scope_for_workspace}; +use crate::events::ids::{ClaimId, SourceId}; +use crate::semantics::pipeline::{ + receipt_view, CandidateCursor, ClaimStatus as SemanticClaimStatus, + ClaimView as SemanticClaimView, ParallelRelateOptions, RelateOutcome, RelateTemporalPolicy, + RelateThresholds, +}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +pub(super) struct SemanticRelateOutput { + pub(super) outcome: crate::semantics::pipeline::RelateOutcome, + pub(super) cache_keys: BTreeMap<(String, String), String>, + pub(super) judge_fingerprint: String, +} + +#[derive(Clone, Copy)] +pub(super) struct RelateBackendContext<'a> { + pub(super) root: &'a Path, + pub(super) gateway: Option<&'a crate::gateway::GatewayConfig>, + pub(super) claims: &'a [(ClaimId, SemanticClaimView)], + pub(super) thresholds: RelateThresholds, + pub(super) settled: &'a BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + pub(super) temporal: &'a RelateTemporalPolicy, + pub(super) budget: std::time::Duration, + pub(super) candidate_pair_budget: usize, + pub(super) candidate_cursor: CandidateCursor, +} + +#[cfg(feature = "openrouter")] +pub(super) fn relate_with_backends( + context: RelateBackendContext<'_>, +) -> Result { + use crate::extract::cache::CachingRelater; + use crate::semantics::openrouter::{OpenRouterEmbedder, OpenRouterRelater}; + use crate::semantics::pipeline::relate_claims_settled_parallel_temporal; + use crate::semantics::ClaimRelater as _; + + let RelateBackendContext { + root, + gateway, + claims, + thresholds, + settled, + temporal, + budget, + candidate_pair_budget, + candidate_cursor, + } = context; + let embedder = OpenRouterEmbedder::new(None, gateway).map_err(semantic_error)?; + let cache_dir = std::env::var_os(ENV_RELATE_CACHE) + .map_or_else(|| root.join(DEFAULT_RELATE_CACHE), PathBuf::from); + let caching_relater = CachingRelater::new( + OpenRouterRelater::new(None, gateway).map_err(semantic_error)?, + cache_dir, + ); + let judge_fingerprint = caching_relater.fingerprint(); + // Judge calls are independent network waits; fan out across workers and + // reassemble in pair order so settlement stays byte-identical. 4 default + // workers keeps provider pressure polite; clamp guards misconfiguration. + let concurrency = std::env::var("TEXO_RELATE_CONCURRENCY") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(4) + .clamp(1, 16); + let relation_output = relate_claims_settled_parallel_temporal( + claims, + &embedder, + &caching_relater, + thresholds, + settled, + ParallelRelateOptions { + temporal, + budget, + concurrency, + candidate_pair_budget, + candidate_cursor, + }, + ) + .map_err(semantic_error)?; + let cache_keys = relate_cache_keys(&caching_relater, claims, &relation_output); + Ok(SemanticRelateOutput { + outcome: relation_output, + cache_keys, + judge_fingerprint, + }) +} + +#[cfg(not(feature = "openrouter"))] +pub(super) fn relate_with_backends( + _context: RelateBackendContext<'_>, +) -> Result { + Err(TexoError::Semantics { + backend: "openrouter".to_string(), + detail: "openrouter feature is disabled".to_string(), + }) +} + +#[cfg(feature = "openrouter")] +pub(super) fn semantic_error(error: impl std::error::Error + Send + Sync + 'static) -> TexoError { + TexoError::Semantics { + backend: "openrouter".to_string(), + detail: crate::error::error_chain(&error), + } +} + +pub(super) fn semantic_claims_from_view( + view: &WorkspaceView, +) -> Result, TexoError> { + let receipts = claim_record_receipts()?; + let mut claims = Vec::new(); + for claim in &view.claims { + if claim.status != crate::claims::status::ClaimStatus::Current { + continue; + } + let claim_id = ClaimId::try_from(claim.card.claim_id.as_str())?; + let source_id = SourceId::try_from(claim.card.source_id.as_str())?; + let receipt = + receipts + .get(&claim.card.claim_id) + .ok_or_else(|| TexoError::MissingEntity { + entity: entity_for_claim(&claim.card.claim_id), + })?; + let supersedes = claim + .supersedes + .iter() + .map(|id| ClaimId::try_from(id.as_str())) + .collect::, _>>()?; + let superseded_by = claim + .card + .superseded_by + .as_deref() + .map(ClaimId::try_from) + .transpose()?; + claims.push(( + claim_id.clone(), + SemanticClaimView { + claim_id, + workspace_id: claim.card.workspace_id.clone(), + source_id, + source_path: claim.card.source_path.clone(), + line_start: claim.card.line_start, + line_end: claim.card.line_end, + text: claim.card.text.clone(), + normalized_text: claim.card.normalized_text.clone(), + subject_hint: claim.card.subject_hint.clone().unwrap_or_default(), + predicate_hint: claim.card.predicate_hint.clone().unwrap_or_default(), + object_hint: claim.card.object_hint.clone().unwrap_or_default(), + confidence_ppm: claim.card.confidence_ppm, + extractor_kind: claim.card.extractor_kind.clone(), + status: SemanticClaimStatus::Current, + receipt: receipt_view( + 0, + receipt.sequence, + "ClaimRecorded", + &scope_for_workspace(&view.workspace_id), + &entity_for_claim(&claim.card.claim_id), + ), + supersedes, + superseded_by, + }, + )); + } + claims.sort_by(|left, right| { + left.1 + .receipt + .sequence + .get() + .cmp(&right.1.receipt.sequence.get()) + .then_with(|| left.0.as_str().cmp(right.0.as_str())) + }); + Ok(claims) +} + +#[cfg(feature = "openrouter")] +fn relate_cache_keys( + caching_relater: &crate::extract::cache::CachingRelater, + claims: &[(ClaimId, SemanticClaimView)], + relation_output: &RelateOutcome, +) -> BTreeMap<(String, String), String> { + let by_id = claims + .iter() + .map(|(id, view)| (id.to_string(), view)) + .collect::>(); + relation_output + .judgments() + .iter() + .filter_map(|judgment| { + let old_view = by_id.get(judgment.older_claim.as_str())?; + let new_view = by_id.get(judgment.newer_claim.as_str())?; + Some(( + ( + judgment.older_claim.to_string(), + judgment.newer_claim.to_string(), + ), + caching_relater.cache_key(&old_view.text, &new_view.text), + )) + }) + .collect() +} diff --git a/src/ops/handlers/relate/campaign.rs b/src/ops/handlers/relate/campaign.rs new file mode 100644 index 0000000..953d71c --- /dev/null +++ b/src/ops/handlers/relate/campaign.rs @@ -0,0 +1,412 @@ +use super::super::common::{append_json, semantic_temporal_policy}; +use super::{RelateConflictRow, RelateSupersessionRow, RELATE_PREFILTER}; +use crate::claims::campaign::CampaignCard; +use crate::claims::status::ClaimStatus; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::coordinate::entity_for_relation_campaign; +use crate::events::ids::WorkspaceId; +use crate::events::payloads::RelationCampaignCheckpointV1; +use crate::gateway::{ + resolve_role_with_environment, GatewayConfig, GatewayEnvironment, ModelRole, RoleOverrides, + ENV_BASE_URL, +}; +use crate::ops::env; +use crate::relate::settlement::CampaignPhase; +use crate::semantics::pipeline::{CandidateCursor, RelateTemporalPolicy}; +use batpak::event::{EventPayload, EventSourced}; +use std::collections::BTreeMap; + +const CANDIDATE_POLICY_VERSION: &str = "texo.relate.candidates.v1"; +const CAMPAIGN_BASIS_VERSION: &str = "texo.relate.campaign-basis.v1"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CampaignBasis { + claims: BTreeMap, + temporal_identity_digest_hex: String, +} + +impl CampaignBasis { + pub(super) fn from_view(view: &WorkspaceView, temporal: &RelateTemporalPolicy) -> Self { + Self::from_entries( + view.claims.iter().map(|claim| { + ( + claim.card.claim_id.clone(), + claim.status == ClaimStatus::Current, + ) + }), + temporal, + ) + } + + fn from_entries( + entries: impl IntoIterator, + temporal: &RelateTemporalPolicy, + ) -> Self { + Self { + claims: entries.into_iter().collect(), + temporal_identity_digest_hex: temporal.campaign_identity_digest(), + } + } + + pub(super) fn digest(&self) -> String { + let mut hasher = blake3::Hasher::new(); + hash_field(&mut hasher, CAMPAIGN_BASIS_VERSION.as_bytes()); + for (claim_id, eligible) in &self.claims { + hash_field(&mut hasher, claim_id.as_bytes()); + hasher.update(&[u8::from(*eligible)]); + } + hash_field(&mut hasher, self.temporal_identity_digest_hex.as_bytes()); + hasher.finalize().to_hex().to_string() + } + + pub(super) fn after_publication( + &self, + supersessions: &[RelateSupersessionRow], + conflicts: &[RelateConflictRow], + ) -> Self { + let mut result = self.clone(); + for row in supersessions { + result.claims.insert(row.old_claim_id.clone(), false); + } + for row in conflicts { + result.claims.insert(row.claim_a.clone(), false); + result.claims.insert(row.claim_b.clone(), false); + } + result + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CampaignStart { + AlreadyComplete, + Run(CandidateCursor), +} + +pub(super) fn resolve_start( + latest: Option<&RelationCampaignCheckpointV1>, + basis_digest: &str, + policy_digest: &str, + requested: Option, +) -> Result { + let matching = latest.filter(|checkpoint| { + checkpoint.result_basis_digest_hex == basis_digest + && checkpoint.candidate_policy_digest_hex == policy_digest + }); + match matching.map(|checkpoint| checkpoint.phase) { + Some(CampaignPhase::Complete) => Ok(CampaignStart::AlreadyComplete), + Some(CampaignPhase::Partial { + next_candidate_cursor, + }) => match requested { + None => Ok(CampaignStart::Run(CandidateCursor::from_offset( + next_candidate_cursor, + ))), + Some(cursor) if cursor.offset() == 0 || cursor.offset() == next_candidate_cursor => Ok( + CampaignStart::Run(CandidateCursor::from_offset(next_candidate_cursor)), + ), + Some(cursor) => Err(cursor_mismatch(cursor.offset(), next_candidate_cursor)), + }, + None => match requested { + None => Ok(CampaignStart::Run(CandidateCursor::start())), + Some(cursor) if cursor.offset() == 0 => Ok(CampaignStart::Run(cursor)), + Some(cursor) => Err(TexoError::OpInput { + op: "texo.relate.run".to_string(), + detail: format!( + "candidate cursor {} has no matching durable campaign; restart from cursor 0", + cursor.offset() + ), + }), + }, + } +} + +pub(super) fn latest_checkpoint( + view: &WorkspaceView, +) -> Result, TexoError> { + env::with(|op_env| { + env::deterministic_projection(|| { + let entity = entity_for_relation_campaign(&view.workspace_id); + let mut card = CampaignCard::default(); + for entry in op_env.store.by_entity(&entity) { + if entry.global_sequence() > view.frontier { + break; + } + if entry.event_kind() != ::KIND { + continue; + } + let raw = op_env.store.read_raw(entry.event_id())?; + card.apply_event(&raw.event); + } + Ok::<_, TexoError>(card.latest) + }) + })? +} + +pub(super) fn candidate_policy_digest( + cluster: f32, + prefilter: f32, + gateway: Option<&GatewayConfig>, +) -> String { + let environment = GatewayEnvironment { + base_url: std::env::var(ENV_BASE_URL).ok(), + api_key: None, + model: std::env::var(ModelRole::Embed.model_env()).ok(), + }; + candidate_policy_digest_with_environment(cluster, prefilter, gateway, &environment) +} + +fn candidate_policy_digest_with_environment( + cluster: f32, + prefilter: f32, + gateway: Option<&GatewayConfig>, + environment: &GatewayEnvironment, +) -> String { + let resolved = resolve_role_with_environment( + ModelRole::Embed, + &RoleOverrides::default(), + gateway, + environment, + ); + let mut hasher = blake3::Hasher::new(); + hash_field(&mut hasher, CANDIDATE_POLICY_VERSION.as_bytes()); + hash_field(&mut hasher, &cluster.to_bits().to_be_bytes()); + hash_field(&mut hasher, &prefilter.to_bits().to_be_bytes()); + hash_field(&mut hasher, resolved.provider_id.as_bytes()); + hash_field(&mut hasher, resolved.profile.base_url.as_bytes()); + hash_field(&mut hasher, resolved.config.model.as_bytes()); + hasher.finalize().to_hex().to_string() +} + +pub(super) struct CheckpointDraft { + pub(super) workspace_id: WorkspaceId, + pub(super) evaluated_basis_digest_hex: String, + pub(super) result_basis_digest_hex: String, + pub(super) candidate_policy_digest_hex: String, + pub(super) phase: CampaignPhase, + pub(super) observed_at_ms: u64, +} + +pub(super) fn append_checkpoint( + op: &str, + cx: &mut syncbat::Ctx<'_>, + draft: CheckpointDraft, +) -> Result<(), TexoError> { + append_json( + op, + cx, + ::KIND, + &RelationCampaignCheckpointV1 { + workspace_id: draft.workspace_id, + evaluated_basis_digest_hex: draft.evaluated_basis_digest_hex, + result_basis_digest_hex: draft.result_basis_digest_hex, + candidate_policy_digest_hex: draft.candidate_policy_digest_hex, + phase: draft.phase, + observed_at_ms: draft.observed_at_ms, + }, + ) +} + +struct SettlementGate { + latest: Option, + complete: bool, +} + +fn settlement_gate(view: &WorkspaceView) -> Result { + let (cluster, prefilter, gateway) = env::with(|op_env| { + let semantics = op_env.config.semantics.as_ref(); + let cluster = semantics.map_or_else( + || crate::config::SemanticsConfig::default().cosine_threshold, + |config| config.cosine_threshold, + ); + let prefilter = semantics + .and_then(|config| config.relate_prefilter) + .unwrap_or(RELATE_PREFILTER); + (cluster, prefilter, op_env.config.gateway.clone()) + })?; + let temporal = semantic_temporal_policy(view)?; + let basis_digest = CampaignBasis::from_view(view, &temporal).digest(); + let policy_digest = candidate_policy_digest(cluster, prefilter, gateway.as_ref()); + let latest = latest_checkpoint(view)?; + let complete = latest.as_ref().is_some_and(|checkpoint| { + checkpoint.result_basis_digest_hex == basis_digest + && checkpoint.candidate_policy_digest_hex == policy_digest + && checkpoint.phase == CampaignPhase::Complete + }); + Ok(SettlementGate { latest, complete }) +} + +pub(in crate::ops::handlers) fn settlement_is_complete( + view: &WorkspaceView, +) -> Result { + Ok(settlement_gate(view)?.complete) +} + +pub(in crate::ops::handlers) fn require_complete_settlement( + view: &WorkspaceView, +) -> Result<(), TexoError> { + let gate = settlement_gate(view)?; + if gate.complete { + return Ok(()); + } + let state = gate.latest.as_ref().map_or_else( + || "no durable campaign checkpoint".to_string(), + |checkpoint| match checkpoint.phase { + CampaignPhase::Complete => "checkpoint does not match this exact frontier".to_string(), + CampaignPhase::Partial { + next_candidate_cursor, + } => format!("campaign is partial at candidate cursor {next_candidate_cursor}"), + }, + ); + Err(TexoError::Semantics { + backend: "settlement".to_string(), + detail: format!( + "strict settlement refused authority-bearing output: {state}; run `texo relate` to resume" + ), + }) +} + +fn cursor_mismatch(requested: u64, expected: u64) -> TexoError { + TexoError::OpInput { + op: "texo.relate.run".to_string(), + detail: format!( + "candidate cursor {requested} does not match durable resume cursor {expected}" + ), + } +} + +fn hash_field(hasher: &mut blake3::Hasher, value: &[u8]) { + hasher.update(&(value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn checkpoint(phase: CampaignPhase) -> RelationCampaignCheckpointV1 { + RelationCampaignCheckpointV1 { + workspace_id: WorkspaceId::try_from("workspace").expect("valid workspace"), + evaluated_basis_digest_hex: "a".repeat(64), + result_basis_digest_hex: "a".repeat(64), + candidate_policy_digest_hex: "b".repeat(64), + phase, + observed_at_ms: 1, + } + } + + #[test] + fn campaign_basis_is_order_independent_and_tracks_ineligible_claims() { + let temporal = RelateTemporalPolicy::default(); + let first = CampaignBasis::from_entries( + [ + ("claim-b".to_string(), true), + ("claim-a".to_string(), false), + ], + &temporal, + ); + let reordered = CampaignBasis::from_entries( + [ + ("claim-a".to_string(), false), + ("claim-b".to_string(), true), + ], + &temporal, + ); + let with_new_ineligible = CampaignBasis::from_entries( + [ + ("claim-a".to_string(), false), + ("claim-b".to_string(), true), + ("claim-c".to_string(), false), + ], + &temporal, + ); + assert_eq!(first.digest(), reordered.digest()); + assert_ne!(first.digest(), with_new_ineligible.digest()); + } + + #[test] + fn publication_changes_only_affected_eligibility() { + let temporal = RelateTemporalPolicy::default(); + let basis = CampaignBasis::from_entries( + [ + ("a".to_string(), true), + ("b".to_string(), true), + ("c".to_string(), true), + ], + &temporal, + ); + let result = basis.after_publication( + &[RelateSupersessionRow { + old_claim_id: "a".to_string(), + new_claim_id: "b".to_string(), + reason: String::new(), + cache_key: String::new(), + }], + &[RelateConflictRow { + conflict_id: "conflict".to_string(), + claim_a: "b".to_string(), + claim_b: "c".to_string(), + reason: String::new(), + cache_key: String::new(), + }], + ); + assert_eq!( + result.claims.values().copied().collect::>(), + [false; 3] + ); + } + + #[test] + fn temporal_evidence_changes_campaign_basis() { + let empty = RelateTemporalPolicy::default(); + let mut rebound = RelateTemporalPolicy::default(); + rebound.bind_claim( + &crate::events::ids::ClaimId::try_from("claim_aaaaaaaaaaaa").expect("valid claim"), + &crate::knowledge::SourceSnapshotId::derive("snapshot"), + ); + let claims = [("claim_aaaaaaaaaaaa".to_string(), true)]; + let first = CampaignBasis::from_entries(claims.clone(), &empty); + let second = CampaignBasis::from_entries(claims, &rebound); + assert_ne!(first.digest(), second.digest()); + } + + #[test] + fn durable_cursor_is_resumed_and_mismatches_are_rejected() { + let partial = checkpoint(CampaignPhase::Partial { + next_candidate_cursor: 42, + }); + let resumed = resolve_start(Some(&partial), &"a".repeat(64), &"b".repeat(64), None) + .expect("durable cursor resumes"); + assert_eq!( + resumed, + CampaignStart::Run(CandidateCursor::from_offset(42)) + ); + assert!(resolve_start( + Some(&partial), + &"a".repeat(64), + &"b".repeat(64), + Some(CandidateCursor::from_offset(7)) + ) + .is_err()); + let complete = checkpoint(CampaignPhase::Complete); + let completed = resolve_start(Some(&complete), &"a".repeat(64), &"b".repeat(64), None) + .expect("complete campaign resolves"); + assert_eq!(completed, CampaignStart::AlreadyComplete); + } + + #[test] + fn candidate_policy_digest_excludes_api_key() { + let first = GatewayEnvironment { + base_url: Some("https://models.example/v1".to_string()), + api_key: Some("first-secret".to_string()), + model: Some("embed-model".to_string()), + }; + let second = GatewayEnvironment { + api_key: Some("second-secret".to_string()), + ..first.clone() + }; + assert_eq!( + candidate_policy_digest_with_environment(0.8, 0.6, None, &first), + candidate_policy_digest_with_environment(0.8, 0.6, None, &second) + ); + } +} diff --git a/src/ops/handlers/relate/publication.rs b/src/ops/handlers/relate/publication.rs new file mode 100644 index 0000000..226030b --- /dev/null +++ b/src/ops/handlers/relate/publication.rs @@ -0,0 +1,204 @@ +use super::super::common::append_json; +use super::backend::SemanticRelateOutput; +use super::{RelateConflictRow, RelateSupersessionRow}; +use crate::error::TexoError; +use crate::events::coordinate::{entity_for_claim, entity_for_conflict}; +use crate::events::ids::WorkspaceId; +use crate::events::machines::{ + transition_record, TransitionCauseV1, CLAIM_MACHINE, CONFLICT_MACHINE, +}; +use crate::events::payloads::{ + ClaimSupersededV2, ConflictOpenedV2, RelationDeferredV1, RelationJudgedV1, +}; +use crate::semantics::pipeline::RelateOutcome; +use crate::semantics::score::unit_interval_to_ppm; +use batpak::event::EventPayload; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) fn append_relation_judgments( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + workspace_id: &WorkspaceId, + related: &SemanticRelateOutput, + observed_at_ms: u64, +) -> Result<(), TexoError> { + for judgment in related.outcome.judgments() { + if judgment.reused_authority { + continue; + } + let cache_key_hex = related + .cache_keys + .get(&( + judgment.older_claim.to_string(), + judgment.newer_claim.to_string(), + )) + .cloned() + .unwrap_or_default(); + append_json( + op, + cx, + ::KIND, + &RelationJudgedV1 { + workspace_id: workspace_id.clone(), + older_claim: judgment.older_claim.clone(), + newer_claim: judgment.newer_claim.clone(), + relation: judgment.verdict.relation.into(), + score_ppm: unit_interval_to_ppm(judgment.verdict.score), + judge_fingerprint: related.judge_fingerprint.clone(), + cache_key_hex, + observed_at_ms, + }, + )?; + } + Ok(()) +} + +pub(super) fn append_relation_deferrals( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + workspace_id: &WorkspaceId, + unresolved: &[crate::relate::settlement::UnresolvedPair], + observed_at_ms: u64, +) -> Result<(), TexoError> { + for unresolved in unresolved { + append_json( + op, + cx, + ::KIND, + &RelationDeferredV1 { + workspace_id: workspace_id.clone(), + older_claim: unresolved.old_claim.clone(), + newer_claim: unresolved.new_claim.clone(), + failure_class: unresolved.failure.class, + attempts: unresolved.failure.attempts, + observed_at_ms, + }, + )?; + } + Ok(()) +} + +pub(super) struct RelatePublicationPlan { + pub(super) supersessions: Vec, + pub(super) conflicts: Vec, +} + +pub(super) fn relate_publication(outcome: &RelateOutcome) -> RelatePublicationPlan { + match outcome { + RelateOutcome::Complete(complete) => RelatePublicationPlan { + supersessions: complete.related.supersessions.clone(), + conflicts: complete.related.conflicts.clone(), + }, + RelateOutcome::Partial(_) => RelatePublicationPlan { + supersessions: Vec::new(), + conflicts: Vec::new(), + }, + } +} + +pub(super) fn append_relate_supersessions( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + workspace_id: &str, + decisions: &[crate::semantics::pipeline::SupersessionEdge], + cache_keys: &BTreeMap<(String, String), String>, + observed_at_ms: u64, +) -> Result, TexoError> { + let mut rows = Vec::new(); + for (old, new, reason) in decisions { + let old_id = old.to_string(); + let new_id = new.to_string(); + let cache_key = cache_keys + .get(&(old_id.clone(), new_id.clone())) + .cloned() + .unwrap_or_default(); + append_json( + op, + cx, + ::KIND, + &ClaimSupersededV2 { + old_claim_id: old_id.clone(), + new_claim_id: new_id.clone(), + workspace_id: workspace_id.to_string(), + reason: reason.clone(), + decided_by: "texo-relate".to_string(), + observed_at_ms, + transition: transition_record( + CLAIM_MACHINE, + &entity_for_claim(&old_id), + 1, + 2, + vec![TransitionCauseV1 { + lane: 0, + key: format!("relate:{cache_key}"), + }], + observed_at_ms, + ), + }, + )?; + rows.push(RelateSupersessionRow { + old_claim_id: old_id, + new_claim_id: new_id, + reason: reason.clone(), + cache_key, + }); + } + Ok(rows) +} + +pub(super) fn append_relate_conflicts( + op: &'static str, + cx: &mut syncbat::Ctx<'_>, + workspace_id: &str, + decisions: &[crate::semantics::pipeline::ConflictEntry], + existing: &BTreeSet, + cache_keys: &BTreeMap<(String, String), String>, + observed_at_ms: u64, +) -> Result, TexoError> { + let mut rows = Vec::new(); + for conflict in decisions { + let conflict_id = conflict.conflict_id.to_string(); + if existing.contains(&conflict_id) { + continue; + } + let claim_a = conflict.claim_a.to_string(); + let claim_b = conflict.claim_b.to_string(); + let cache_key = cache_keys + .get(&(claim_a.clone(), claim_b.clone())) + .cloned() + .unwrap_or_default(); + append_json( + op, + cx, + ::KIND, + &ConflictOpenedV2 { + conflict_id: conflict_id.clone(), + workspace_id: workspace_id.to_string(), + claim_a: claim_a.clone(), + claim_b: claim_b.clone(), + reason: conflict.reason.clone(), + detector: "texo-relate".to_string(), + observed_at_ms, + transition: transition_record( + CONFLICT_MACHINE, + &entity_for_conflict(&conflict_id), + 0, + 1, + vec![TransitionCauseV1 { + lane: 0, + key: format!("relate:{cache_key}"), + }], + observed_at_ms, + ), + }, + )?; + rows.push(RelateConflictRow { + conflict_id, + claim_a, + claim_b, + reason: conflict.reason.clone(), + cache_key, + }); + } + Ok(rows) +} diff --git a/src/ops/handlers/render.rs b/src/ops/handlers/render.rs new file mode 100644 index 0000000..f3b3aaa --- /dev/null +++ b/src/ops/handlers/render.rs @@ -0,0 +1,378 @@ +use super::agent_context::AgentContextOutput; +use super::common::{ + assemble_snapshot_view, claim_receipt, op_runtime, parse_input, run_op, + WORKSPACE_VIEW_PROJECTION, +}; +use super::ingest::resolve_path; +use super::model::AgentReceiptRow; +use crate::claims::workspace::WorkspaceView; +use crate::error::TexoError; +use crate::events::ids::SourceId; +use crate::extract::markdown::{collect_markdown_files, MarkdownDocument}; +use crate::extract::normalize::normalize_line; +use crate::knowledge::SnapshotRead; +use crate::ops::env; +use crate::relate::heuristic; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = STALENESS_CHECK, + register = register_staleness_check, + register_item = staleness_check_item, + name = "texo.staleness.check", + effect = Inspect, + input_schema = "texo.staleness.check.input.v3", + output_schema = "texo.staleness.check.output.v3", + receipt_kind = "receipt.texo.staleness.check.v3", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn staleness_check(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.staleness.check", || { + let input: StalenessCheckInput = parse_input("texo.staleness.check", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.staleness.check", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + let (root, workspace_id) = + env::with(|op_env| (op_env.root.clone(), op_env.workspace_id.clone()))?; + let path = resolve_path(&root, &input.path); + check_staleness_from_view(&view, &workspace_id, &root, &path, snapshot) + }) +} +#[derive(Debug, Deserialize)] +struct StalenessCheckInput { + path: PathBuf, + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +pub(super) struct StalenessReport { + workspace_id: String, + checked_path: String, + replayed_through_sequence: u64, + diagnostics: Vec, + snapshot: SnapshotRead, +} + +impl StalenessReport { + pub(super) fn empty( + workspace_id: String, + replayed_through_sequence: u64, + snapshot: SnapshotRead, + ) -> Self { + Self { + workspace_id, + checked_path: ".".to_string(), + replayed_through_sequence, + diagnostics: Vec::new(), + snapshot, + } + } +} + +#[derive(Debug, Serialize)] +struct StaleDiagnostic { + file: String, + line_start: u32, + line_end: u32, + severity: DiagnosticSeverity, + message: String, + claim_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + superseded_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + receipt: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "lowercase")] +enum DiagnosticSeverity { + Warning, +} + +#[derive(Debug, Serialize)] +struct DiagnosticSource { + path: String, + line_start: u32, +} + +pub(super) fn render_onboarding(context: &AgentContextOutput) -> String { + let mut out = String::from("# Generated Onboarding\n\n"); + out.push_str( + "_This document is a projection replayed from the texo claim-chain. \ + It is not source truth._\n\n", + ); + writeln!( + &mut out, + "_Replayed through local store sequence {}._\n", + context.replayed_through_sequence + ) + .expect("writing to a String cannot fail"); + out.push_str("## Current claims\n\n"); + for claim in &context.claims { + writeln!( + &mut out, + "- **{}** ({}): {} \n _source: {}:{}_", + claim.claim_id, + claim.subject_hint.clone().unwrap_or_default(), + claim.text, + claim.source.path, + claim.source.line_start + ) + .expect("writing to a String cannot fail"); + } + if !context.stale_claims.is_empty() { + out.push_str("\n## Stale claims (do not trust)\n\n"); + for stale in &context.stale_claims { + writeln!( + &mut out, + "- {}: \"{}\" superseded by {}", + stale.claim_id, stale.text, stale.superseded_by + ) + .expect("writing to a String cannot fail"); + } + } + if !context.conflicts.is_empty() { + out.push_str("\n## Conflicts (unresolved — both claimed, neither wins)\n\n"); + for conflict in &context.conflicts { + writeln!( + &mut out, + "- \"{}\" ({}) vs \"{}\" ({})", + conflict.claim_a_text, conflict.claim_a, conflict.claim_b_text, conflict.claim_b + ) + .expect("writing to a String cannot fail"); + } + } + out +} + +pub(super) fn render_index_html( + context: &AgentContextOutput, + stale: &StalenessReport, + conflicts: &heuristic::ConflictReport, +) -> Result { + let mut claim_cards = String::new(); + for claim in &context.claims { + let supersedes = if claim.supersedes.is_empty() { + String::new() + } else { + format!( + "

supersedes: {}

", + claim.supersedes.join(", ") + ) + }; + write!( + &mut claim_cards, + r#"
+

Claim {id}

+

status: current

+

subject: {subject}

+

local sequence: {seq}

+

frontier: replayed through seq {frontier}

+

source: {path}:{line}

+

receipt: {receipt}

+ {supersedes} +
{text}
+
"#, + id = claim.claim_id, + subject = claim.subject_hint.clone().unwrap_or_default(), + seq = claim.receipt.sequence, + frontier = context.replayed_through_sequence, + path = claim.source.path, + line = claim.source.line_start, + receipt = claim.receipt.event_id, + supersedes = supersedes, + text = html_escape(&claim.text), + ) + .expect("writing to a String cannot fail"); + } + let mut stale_cards = String::new(); + for diagnostic in &stale.diagnostics { + write!( + &mut stale_cards, + r#"
+

Stale line {}:{}

+

{}

+
"#, + diagnostic.file, + diagnostic.line_start, + html_escape(&diagnostic.message) + ) + .expect("writing to a String cannot fail"); + } + let conflicts_json = serde_json::to_string_pretty(conflicts)?; + Ok(format!( + r#" + + + + texo claim explorer + + + +
+

A block explorer for stale team beliefs.

+

Every claim below was replayed from a BatPak journal. The generated onboarding doc is a projection, not source truth.

+
+
+

Current claims

+ {claim_cards} +
+
+

Stale diagnostics

+ {stale_cards} +
+
+

Conflicts ({conflict_count})

+ 
{conflicts_json}
+
+
+ texo uses one local BatPak journal. Sequences are per-store. No global order, network consensus, or distributed replication is claimed. +
+ +"#, + conflict_count = conflicts.conflicts.len(), + conflicts_json = html_escape(&conflicts_json) + )) +} + +fn html_escape(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +pub(super) fn check_staleness_from_view( + view: &WorkspaceView, + workspace_id: &str, + root: &Path, + input: &Path, + snapshot: SnapshotRead, +) -> Result { + let checked_path = input + .strip_prefix(root) + .unwrap_or(input) + .to_string_lossy() + .to_string(); + let discovery = collect_markdown_files(input).map_err(|error| TexoError::Source { + path: input.to_string_lossy().to_string(), + detail: error.to_string(), + })?; + if let Some(failure) = discovery.failures.first() { + return Err(TexoError::Source { + path: failure.path.to_string_lossy().to_string(), + detail: failure.error.to_string(), + }); + } + let by_id = view + .claims + .iter() + .map(|claim| (claim.card.claim_id.clone(), claim)) + .collect::>(); + let mut diagnostics = Vec::new(); + for path in discovery.files { + diagnostics.extend(stale_diagnostics_for_path(view, &by_id, root, &path)?); + } + Ok(StalenessReport { + workspace_id: workspace_id.to_string(), + checked_path, + replayed_through_sequence: view.frontier, + diagnostics, + snapshot, + }) +} + +fn stale_diagnostics_for_path( + view: &WorkspaceView, + by_id: &BTreeMap, + root: &Path, + path: &Path, +) -> Result, TexoError> { + let mut diagnostics = Vec::new(); + + let doc = MarkdownDocument::from_path(path, root).map_err(|error| TexoError::Source { + path: path.to_string_lossy().to_string(), + detail: error.to_string(), + })?; + let source_id = SourceId::try_from(doc.source_id.as_str())?; + // Match superseded claims of THIS doc by normalized-text containment in + // the doc's current lines. Reconstructing claim ids from whole lines + // only matches heuristic whole-line claims; LLM extraction proposes + // sub-sentence claims whose identity a line-level rebuild never hits. + // Normalize each doc line once, not once per superseded claim. + let normalized_lines = doc + .lines + .iter() + .map(|line| (line, normalize_line(&line.text))) + .collect::>(); + for claim in &view.claims { + if claim.card.phase != 2 || claim.card.source_id != source_id.as_str() { + continue; + } + let needle = claim.card.normalized_text.as_str(); + if needle.is_empty() { + continue; + } + let line = normalized_lines + .iter() + .find(|(line, normalized)| { + line.number == claim.card.line_start && normalized.contains(needle) + }) + .or_else(|| { + normalized_lines + .iter() + .find(|(_, normalized)| normalized.contains(needle)) + }) + .map(|(line, _)| *line); + let Some(line) = line else { + continue; // the stale text no longer appears in the doc + }; + let superseded_by = claim.card.superseded_by.clone(); + let source = superseded_by + .as_ref() + .and_then(|id| by_id.get(id)) + .map(|superseder| DiagnosticSource { + path: superseder.card.source_path.clone(), + line_start: superseder.card.line_start, + }); + let receipt = superseded_by + .as_ref() + .and_then(|id| claim_receipt(id).ok()) + .or_else(|| claim_receipt(&claim.card.claim_id).ok()); + let message = format!( + "Claim appears stale: superseded by {} at {}.", + superseded_by.as_deref().unwrap_or("unknown"), + receipt.as_ref().map_or_else( + || "unknown seq".to_string(), + |receipt| format!("local seq {}", receipt.sequence) + ) + ); + diagnostics.push(StaleDiagnostic { + file: doc.path.clone(), + line_start: line.number, + line_end: line.number, + severity: DiagnosticSeverity::Warning, + message, + claim_id: claim.card.claim_id.clone(), + superseded_by, + source, + receipt, + }); + } + + Ok(diagnostics) +} diff --git a/src/ops/handlers/stats.rs b/src/ops/handlers/stats.rs new file mode 100644 index 0000000..408acf2 --- /dev/null +++ b/src/ops/handlers/stats.rs @@ -0,0 +1,135 @@ +use super::agent_context::build_agent_context_from_view; +use super::common::{ + assemble_current_view, op_runtime, parse_input, run_op, snapshot_for_view, + WORKSPACE_VIEW_PROJECTION, +}; +use crate::error::TexoError; +use crate::events::coordinate::scope_for_workspace; +use crate::ops::env; +use batpak::coordinate::Region; +use serde::{Deserialize, Serialize}; +use std::path::Path; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = STATS_READ, + register = register_stats_read, + register_item = stats_read_item, + name = "texo.stats.read", + effect = Inspect, + input_schema = "texo.stats.read.input.v1", + output_schema = "texo.stats.read.output.v1", + receipt_kind = "receipt.texo.stats.read.v1", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +pub(super) fn stats_read(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.stats.read", || { + let _input: StatsReadInput = parse_input("texo.stats.read", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.stats.read", error))?; + let view = assemble_current_view()?; + let (root, config, journal) = env::with(|op_env| { + ( + op_env.root.clone(), + op_env.config.clone(), + op_env.journal.clone(), + ) + })?; + let store_path = config.store_path_buf(&root); + let projection_path = root + .join(".texo/cache/workspace-view") + .join(format!("{}--{}.bin", view.workspace_id, journal.id)); + let context = build_agent_context_from_view(&view, None, true, snapshot_for_view(&view)?)?; + let agent_context_bytes = serde_json::to_vec(&context)?.len(); + Ok(StatsReadOutput { + journal_id: journal.id, + journal_role: journal.role, + claims_total: view.claims.len(), + events_total: workspace_event_count()?, + journal_bytes: journal_file_bytes(&store_path)?, + projection_bytes: file_bytes(&projection_path)?, + agent_context_bytes: u64::try_from(agent_context_bytes).unwrap_or(u64::MAX), + frontier_sequence: view.frontier, + }) + }) +} +#[derive(Debug, Deserialize)] +struct StatsReadInput {} +#[derive(Debug, Serialize)] +struct StatsReadOutput { + journal_id: crate::topology::JournalId, + journal_role: crate::topology::JournalRole, + claims_total: usize, + events_total: usize, + journal_bytes: u64, + projection_bytes: u64, + agent_context_bytes: u64, + frontier_sequence: u64, +} +pub(super) fn claim_phase_name(phase: u64) -> &'static str { + match phase { + 0 => "unrecorded", + 1 => "current", + 2 => "superseded", + _ => "invalid", + } +} + +pub(super) fn conflict_phase_name(phase: u64) -> &'static str { + match phase { + 0 => "unopened", + 1 => "open", + 2 => "resolved", + 3 => "ignored", + _ => "invalid", + } +} + +pub(super) fn workspace_event_count() -> Result { + env::with(|op_env| { + let region = Region::scope(scope_for_workspace(&op_env.workspace_id)); + let mut after = None; + let mut count = 0usize; + loop { + let page = op_env.store.query_entries_after(®ion, after, 256); + if page.is_empty() { + break; + } + count = count.saturating_add(page.len()); + after = page.last().map(batpak::store::IndexEntry::global_sequence); + } + count + }) +} + +pub(super) fn file_bytes(path: &Path) -> Result { + match std::fs::metadata(path) { + Ok(metadata) => Ok(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(error) => Err(error.into()), + } +} + +pub(super) fn journal_file_bytes(path: &Path) -> Result { + let metadata = match std::fs::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error.into()), + }; + if metadata.is_file() { + return Ok( + if path.extension().and_then(std::ffi::OsStr::to_str) == Some("fbat") { + metadata.len() + } else { + 0 + }, + ); + } + let mut bytes = 0u64; + for entry in std::fs::read_dir(path)? { + bytes = bytes.saturating_add(journal_file_bytes(&entry?.path())?); + } + Ok(bytes) +} diff --git a/src/ops/handlers/verify.rs b/src/ops/handlers/verify.rs new file mode 100644 index 0000000..7ad5a55 --- /dev/null +++ b/src/ops/handlers/verify.rs @@ -0,0 +1,188 @@ +use super::common::elapsed_ms; +use super::common::{op_runtime, parse_input, run_op, WORKSPACE_VIEW_PROJECTION}; +use crate::claims::workspace::{assemble, WorkspaceView}; +use crate::error::TexoError; +use crate::events::coordinate::scope_for_workspace; +use crate::events::machines::{CLAIM_EDGES, CLAIM_MACHINE, CONFLICT_EDGES, CONFLICT_MACHINE}; +use crate::events::payloads::{ + ClaimEvidenceLinkedV1, ClaimRecordedV2, ClaimSupersededV2, ConflictOpenedV2, + ConflictResolvedV2, EvidenceOccurrenceRecordedV1, EvidenceReconciliationAcceptedV1, + OnboardingCompiledV2, RelationCampaignCheckpointV1, RelationDeferredV1, RelationJudgedV1, + SessionTurnV1, SourceObservedV2, SourceSnapshotRecordedV1, SourceSnapshotRelationV1, + WorkspaceInitializedV2, +}; +use crate::ops::env; +use batpak::coordinate::Region; +use batpak::event::{EventKind, EventPayload}; +use batpak::id::EntityIdType; +use serde::{Deserialize, Serialize}; +use std::time::Instant; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = VERIFY_RUN, + register = register_verify_run, + register_item = verify_run_item, + name = "texo.verify.run", + effect = Inspect, + input_schema = "texo.verify.run.input.v2", + output_schema = "texo.verify.run.output.v2", + receipt_kind = "receipt.texo.verify.run.v2", + reads_events = ["evt.e001", "evt.e002", "evt.e003", "evt.e004", "evt.e005", "evt.e006", "evt.e007", "evt.e008", "evt.e009", "evt.e00a", "evt.e00b", "evt.e00c", "evt.e00d", "evt.e00e", "evt.e00f", "evt.e010", "evt.e012"], + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn verify_run(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.verify.run", || { + let _input: VerifyRunInput = parse_input("texo.verify.run", input)?; + let replay_started = Instant::now(); + for kind in DOMAIN_KINDS { + cx.event_read_handle() + .read_event(format!("evt.{:04x}", kind.as_raw_u16())) + .map_err(|error| op_runtime("texo.verify.run", error))?; + } + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.verify.run", error))?; + + let mut errors = Vec::new(); + let (journal_ok, view, events_replayed) = env::with(|op_env| { + let chain = op_env.store.verify_chain()?; + let mut journal_ok = chain.is_intact(); + if !chain.is_intact() { + errors.push(format!("chain: {chain:?}")); + } + let scope = scope_for_workspace(&op_env.workspace_id); + let region = Region::scope(&scope); + let mut after = None; + let mut events_replayed = 0usize; + loop { + let page = op_env.store.query_entries_after(®ion, after, 256); + if page.is_empty() { + break; + } + for entry in &page { + events_replayed = events_replayed.saturating_add(1); + if !DOMAIN_KINDS.contains(&entry.event_kind()) { + journal_ok = false; + errors.push(format!( + "unknown event kind evt.{:04x} at {}", + entry.event_kind().as_raw_u16(), + entry.global_sequence() + )); + } + if let Err(error) = op_env.store.read_raw(entry.event_id()) { + journal_ok = false; + errors.push(format!( + "decode {}: {error}", + event_id_hex(entry.event_id()) + )); + } + } + after = page.last().map(batpak::store::IndexEntry::global_sequence); + } + let mut cache = op_env.cache.borrow_mut(); + let view = env::deterministic_projection(|| { + assemble(&op_env.store, &op_env.workspace_id, &mut cache) + })?; + Ok::<_, TexoError>((journal_ok, view, events_replayed)) + })??; + + let projection_ok = view + .claims + .iter() + .all(|claim| claim.card.anomalies.is_empty()) + && view + .conflicts + .iter() + .all(|conflict| conflict.anomalies.is_empty()); + if !projection_ok { + errors.push("projection anomalies present".to_string()); + } + let transitions_ok = validate_transition_edges(&view, &mut errors); + + Ok(VerifyRunOutput { + projection_ok, + journal_ok, + transitions_ok, + errors, + replay_ms: elapsed_ms(replay_started), + events_replayed, + }) + }) +} +const DOMAIN_KINDS: [EventKind; 17] = [ + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, + ::KIND, +]; +#[derive(Debug, Deserialize)] +struct VerifyRunInput {} + +#[derive(Debug, Serialize)] +struct VerifyRunOutput { + projection_ok: bool, + journal_ok: bool, + transitions_ok: bool, + errors: Vec, + replay_ms: u64, + events_replayed: usize, +} +fn validate_transition_edges(view: &WorkspaceView, errors: &mut Vec) -> bool { + let mut ok = true; + for claim in &view.claims { + let edge = match claim.card.phase { + 1 => Some((0, 1)), + 2 => Some((1, 2)), + _ => None, + }; + if edge.is_none_or(|edge| !CLAIM_EDGES.contains(&edge)) { + ok = false; + errors.push(format!( + "claim {} invalid phase {} for {CLAIM_MACHINE}", + claim.card.claim_id, claim.card.phase + )); + } + if claim.card.phase == 2 && claim.card.superseded_by.is_none() { + ok = false; + errors.push(format!( + "claim {} superseded without target", + claim.card.claim_id + )); + } + } + for conflict in &view.conflicts { + let edge = match conflict.phase { + 1 => Some((0, 1)), + 2 => Some((1, 2)), + 3 => Some((1, 3)), + _ => None, + }; + if edge.is_none_or(|edge| !CONFLICT_EDGES.contains(&edge)) { + ok = false; + errors.push(format!( + "conflict {} invalid phase {} for {CONFLICT_MACHINE}", + conflict.conflict_id, conflict.phase + )); + } + } + ok +} + +fn event_id_hex(event_id: batpak::id::EventId) -> String { + format!("{:032x}", event_id.as_u128()) +} diff --git a/src/ops/handlers/workspace.rs b/src/ops/handlers/workspace.rs new file mode 100644 index 0000000..1a605ba --- /dev/null +++ b/src/ops/handlers/workspace.rs @@ -0,0 +1,192 @@ +use super::common::{ + append_json, assemble_snapshot_view, config_error, op_runtime, parse_input, run_op, + status_coverage, take_receipts, WORKSPACE_VIEW_PROJECTION, +}; +use super::relate::{authoritative_settlements, settlement_is_complete}; +use crate::config::{TexoRootConfig, WorkspaceEntry}; +use crate::error::TexoError; +use crate::events::coordinate::entity_for_workspace_meta; +use crate::events::payloads::WorkspaceInitializedV2; +use crate::knowledge::{KnowledgeCoverage, SnapshotRead}; +use crate::ops::env; +use crate::ops::env::ReceiptNote; +use batpak::event::EventPayload; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use syncbat::HandlerResult; + +#[syncbat::operation( + descriptor = WORKSPACE_INIT, + register = register_workspace_init, + register_item = workspace_init_item, + name = "texo.workspace.init", + effect = Persist, + input_schema = "texo.workspace.init.input.v2", + output_schema = "texo.workspace.init.output.v2", + receipt_kind = "receipt.texo.workspace.init.v2", + appends_events = ["evt.e007"] +)] +#[tracing::instrument(skip_all)] +fn workspace_init(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.workspace.init", || { + let input: WorkspaceInitInput = parse_input("texo.workspace.init", input)?; + let (root, observed_at_ms) = + env::with(|op_env| (op_env.root.clone(), op_env.observed_at_ms))?; + let config_path = root.join(".texo").join("config.toml"); + + let mut root_config = if config_path.exists() { + TexoRootConfig::load(&config_path).map_err(config_error)? + } else { + TexoRootConfig { + default_workspace: input.workspace_id.clone(), + workspaces: BTreeMap::new(), + gateway: None, + } + }; + root_config + .default_workspace + .clone_from(&input.workspace_id); + if !root_config.workspaces.contains_key(&input.workspace_id) { + root_config.upsert_workspace( + &input.workspace_id, + WorkspaceEntry::for_id(&input.workspace_id), + ); + } + + let raw = toml::to_string_pretty(&root_config).map_err(|error| TexoError::Config { + detail: error.to_string(), + source: Some(Box::new(error)), + })?; + let config_unchanged = std::fs::read(&config_path) + .ok() + .is_some_and(|existing| existing == raw.as_bytes()); + if !config_unchanged { + if let Some(parent) = config_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&config_path, raw.as_bytes())?; + } + let config_digest_hex = blake3::hash(raw.as_bytes()).to_hex().to_string(); + let journal_digest_matches = env::with(|op_env| { + let entity = entity_for_workspace_meta(&input.workspace_id); + let mut entries = op_env.store.by_entity(&entity); + entries.sort_by_key(batpak::store::IndexEntry::global_sequence); + let Some(entry) = entries.last() else { + return Ok::<_, TexoError>(false); + }; + let raw = op_env.store.read_raw(entry.event_id())?; + let payload: WorkspaceInitializedV2 = batpak::encoding::from_bytes(&raw.event.payload) + .map_err(|error| TexoError::Decode { + entity, + detail: error.to_string(), + })?; + Ok(payload.config_digest_hex == config_digest_hex) + })??; + let already_initialized = config_unchanged && journal_digest_matches; + + append_json( + "texo.workspace.init", + cx, + ::KIND, + &WorkspaceInitializedV2 { + workspace_id: input.workspace_id.clone(), + schema: "texo.v2".to_string(), + config_digest_hex, + created_at_ms: observed_at_ms, + }, + )?; + let mut receipts = take_receipts()?; + let receipt = receipts.pop().ok_or_else(|| TexoError::OpRuntime { + op: "texo.workspace.init".to_string(), + detail: "workspace init append produced no receipt".to_string(), + denied: false, + })?; + + Ok(WorkspaceInitOutput { + workspace_id: input.workspace_id, + config_path: config_path.to_string_lossy().to_string(), + already_initialized, + receipt, + }) + }) +} +#[syncbat::operation( + descriptor = WORKSPACE_STATUS, + register = register_workspace_status, + register_item = workspace_status_item, + name = "texo.workspace.status", + effect = Inspect, + input_schema = "texo.workspace.status.input.v2", + output_schema = "texo.workspace.status.output.v2", + receipt_kind = "receipt.texo.workspace.status.v2", + queries_projections = ["texo.workspace.view.v2"] +)] +#[tracing::instrument(skip_all)] +fn workspace_status(input: &[u8], cx: &mut syncbat::Ctx<'_>) -> HandlerResult { + run_op("texo.workspace.status", || { + let input: WorkspaceStatusInput = parse_input("texo.workspace.status", input)?; + cx.projection_read_handle() + .query_projection(WORKSPACE_VIEW_PROJECTION) + .map_err(|error| op_runtime("texo.workspace.status", error))?; + let (view, snapshot) = assemble_snapshot_view(input.snapshot.as_deref())?; + let settlement = authoritative_settlements(Some(view.frontier))?; + let unresolved_pairs = settlement.unresolved_pairs; + let settlement_complete = settlement_is_complete(&view)?; + let (coverage, code_index_available) = status_coverage(&view, &snapshot)?; + let journal = env::with(|op_env| op_env.journal.clone())?; + Ok(WorkspaceStatusOutput { + workspace_id: view.workspace_id.clone(), + journal_id: journal.id, + journal_role: journal.role, + source_journal: journal.source_journal, + replica_mode: journal.replica_mode, + frontier: view.frontier, + freshness: view.freshness, + claims_total: view.claims.len(), + open_conflicts: view.conflicts.iter().filter(|card| card.phase == 1).count(), + settlement_complete, + unresolved_pairs, + authority_warnings: settlement.warnings.len(), + code_index_available, + coverage, + snapshot, + }) + }) +} +#[derive(Debug, Deserialize)] +struct WorkspaceInitInput { + workspace_id: String, +} + +#[derive(Debug, Deserialize)] +struct WorkspaceStatusInput { + #[serde(default)] + snapshot: Option, +} + +#[derive(Debug, Serialize)] +struct WorkspaceStatusOutput { + workspace_id: String, + journal_id: crate::topology::JournalId, + journal_role: crate::topology::JournalRole, + source_journal: Option, + replica_mode: Option, + frontier: u64, + freshness: crate::claims::workspace::ProjectionFreshness, + claims_total: usize, + open_conflicts: usize, + settlement_complete: bool, + unresolved_pairs: usize, + authority_warnings: usize, + code_index_available: bool, + snapshot: SnapshotRead, + coverage: KnowledgeCoverage, +} + +#[derive(Debug, Serialize)] +struct WorkspaceInitOutput { + workspace_id: String, + config_path: String, + already_initialized: bool, + receipt: ReceiptNote, +} diff --git a/src/reconcile.rs b/src/reconcile.rs index bdb8bd7..12a45c1 100644 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -11,58 +11,12 @@ use crate::knowledge::{ use crate::relate::settlement::{PairFailureView, RelationFailureClass}; use crate::semantics::{ClaimRelater, ClaimRelation, RelationVerdict}; -/// Versioned deterministic policy and prompt context. -pub const POLICY_VERSION: &str = "evidence-reconcile-v1"; - -/// Bounds on candidate generation before any paid proposal call. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ReconcileLimits { - /// Maximum candidates retained for one claim. - pub per_claim: usize, - /// Maximum candidates retained for the complete operation. - pub total: usize, - /// Minimum accepted model score in parts per million. - pub min_score_ppm: u32, -} +mod domain; -impl Default for ReconcileLimits { - fn default() -> Self { - Self { - per_claim: 4, - total: 256, - min_score_ppm: 700_000, - } - } -} +pub use domain::{ReconcileCandidate, ReconcileClaim, ReconcileLimits}; -/// Minimal semantic claim view consumed by candidate generation. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ReconcileClaim { - /// Durable claim identity. - pub claim_id: ClaimId, - /// Exact assertion text sent to the proposal model. - pub text: String, - /// Optional deterministic subject hint. - pub subject_hint: String, - /// Optional deterministic predicate hint. - pub predicate_hint: String, - /// Optional deterministic object hint. - pub object_hint: String, -} - -/// One bounded claim/code pair eligible for a cached model proposal. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ReconcileCandidate { - /// Claim receiving evidence if policy accepts the proposal. - pub claim_id: ClaimId, - /// Exact semantic assertion supplied as the proposal's first assertion. - pub claim_text: String, - /// Exact durable occurrence constructed from the disposable code index. - pub occurrence: EvidenceOccurrence, - /// Role-labelled code text supplied as the proposal's second assertion. - pub code_prompt: String, - rank: usize, -} +/// Versioned deterministic policy and prompt context. +pub const POLICY_VERSION: &str = "evidence-reconcile-v1"; /// One cached-or-live proposal ready for deterministic policy evaluation. #[derive(Debug, Clone)] @@ -602,13 +556,8 @@ const STOP_WORDS: &[&str] = &[ "and", "are", "for", "from", "into", "not", "our", "the", "this", "that", "use", "uses", "with", ]; -#[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "score is clamped to the closed 0..=1 interval before ppm conversion" -)] fn score_to_ppm(score: f32) -> u32 { - (score.clamp(0.0, 1.0) * 1_000_000.0).round() as u32 + crate::semantics::score::unit_interval_to_ppm(score) } #[cfg(test)] diff --git a/src/reconcile/domain.rs b/src/reconcile/domain.rs new file mode 100644 index 0000000..3b9fcf9 --- /dev/null +++ b/src/reconcile/domain.rs @@ -0,0 +1,52 @@ +use crate::events::ids::ClaimId; +use crate::knowledge::EvidenceOccurrence; + +/// Bounds on candidate generation before any paid proposal call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReconcileLimits { + /// Maximum candidates retained for one claim. + pub per_claim: usize, + /// Maximum candidates retained for the complete operation. + pub total: usize, + /// Minimum accepted model score in parts per million. + pub min_score_ppm: u32, +} + +impl Default for ReconcileLimits { + fn default() -> Self { + Self { + per_claim: 4, + total: 256, + min_score_ppm: 700_000, + } + } +} + +/// Minimal semantic claim view consumed by candidate generation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReconcileClaim { + /// Durable claim identity. + pub claim_id: ClaimId, + /// Exact assertion text sent to the proposal model. + pub text: String, + /// Optional deterministic subject hint. + pub subject_hint: String, + /// Optional deterministic predicate hint. + pub predicate_hint: String, + /// Optional deterministic object hint. + pub object_hint: String, +} + +/// One bounded claim/code pair eligible for a cached model proposal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReconcileCandidate { + /// Claim receiving evidence if policy accepts the proposal. + pub claim_id: ClaimId, + /// Exact semantic assertion supplied as the proposal's first assertion. + pub claim_text: String, + /// Exact durable occurrence constructed from the disposable code index. + pub occurrence: EvidenceOccurrence, + /// Role-labelled code text supplied as the proposal's second assertion. + pub code_prompt: String, + pub(super) rank: usize, +} diff --git a/src/relate/settlement.rs b/src/relate/settlement.rs index 26bf799..616c1ed 100644 --- a/src/relate/settlement.rs +++ b/src/relate/settlement.rs @@ -4,6 +4,19 @@ use serde::{Deserialize, Serialize}; use crate::events::ids::ClaimId; +/// Durable completion state for one bounded relation campaign. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum CampaignPhase { + /// More candidate slots or a failed pair must be resumed. + Partial { + /// Exact global candidate cursor required by the next page. + next_candidate_cursor: u64, + }, + /// Every required candidate slot and verdict is settled. + Complete, +} + /// Closed relation verdict recorded for a logical pair. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] diff --git a/src/replication.rs b/src/replication.rs index f9afe2c..a1ca6d9 100644 --- a/src/replication.rs +++ b/src/replication.rs @@ -1,29 +1,13 @@ //! Scale-out replica circuits composed from BatPak lifecycle and import APIs. -use std::collections::BTreeSet; -use std::fs::{self, OpenOptions}; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::time::Duration; +use std::path::PathBuf; -use batpak::coordinate::Coordinate; -use batpak::event::EventPayload; -use batpak::id::IdempotencyKey; -use batpak::store::{ - AppendOptions, BatchAppendItem, CausationRef, Open, ReadOnly, Store, StoreConfig, StoreState, -}; use serde::{Deserialize, Serialize}; -use crate::compat::batpak as substrate; -use crate::config::{TexoRootConfig, WorkspaceConfig}; -use crate::error::{Committed, ReplicationFailureKind, TexoError}; -use crate::events::payloads::{ReplicaBatchMaterializedV1, ReplicaSourceEventV1}; -use crate::replica_net::{self, PageRequest, PageResponse}; -use crate::topology::{JournalRole, ReplicaMode, ResolvedJournal}; +use crate::config::WorkspaceConfig; +use crate::topology::ResolvedJournal; -const STATE_SCHEMA_VERSION: u32 = 2; -const READER_REFRESH_ATTEMPTS: u32 = 50; -const READER_REFRESH_BACKOFF: Duration = Duration::from_millis(40); +pub(super) const STATE_SCHEMA_VERSION: u32 = 2; /// Durable operational cursor for one imported read model. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -109,1050 +93,9 @@ struct Circuit { replica_path: PathBuf, } -#[derive(Default)] -struct RemoteProgress { - after: Option, - after_anchor: Option, - source_ceiling: Option, - source_ceiling_anchor: Option, - imported: u64, - deduplicated: u64, - skipped_reserved: u64, - skipped_operational: u64, -} - -/// Bootstrap the configured replica according to its frozen mode. -/// -/// Exact forks preserve event identities at one point in time. Imported read -/// models receive destination-local event ids plus an atomic replica ledger. -/// Existing destination bytes are never cleared by Texo. -/// -/// # Errors -/// Fails closed on malformed topology, non-fresh destinations, `BatPak` errors, -/// verification failures, or evidence persistence failures. -pub fn bootstrap( - root: &Path, - workspace_id: Option<&str>, - replica_id: &str, -) -> Result { - let circuit = resolve_circuit(root, workspace_id, replica_id)?; - ensure_fresh_destination(&circuit.replica_path)?; - match circuit.replica.replica_mode { - Some(ReplicaMode::ExactFork) => bootstrap_exact(root, &circuit), - Some(ReplicaMode::ImportedReadModel) => bootstrap_imported(root, &circuit), - None => Err(replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - "replica declaration has no materialization mode", - )), - } -} - -/// Resume one configured imported read model from its verified source cursor. -/// -/// # Errors -/// Fails closed when the circuit is not an imported read model, its cursor is -/// missing/mismatched, the source anchor changed, or import/verification fails. -pub fn follow_once( - root: &Path, - workspace_id: Option<&str>, - replica_id: &str, -) -> Result { - let circuit = resolve_circuit(root, workspace_id, replica_id)?; - if circuit.replica.replica_mode != Some(ReplicaMode::ImportedReadModel) { - return Err(replication_error( - ReplicationFailureKind::ModeMismatch, - Committed::No, - "only imported_read_model replicas can follow a changing source", - )); - } - let cursor = load_cursor(root, &circuit.workspace.workspace_id, replica_id)?; - validate_cursor_binding(&cursor, &circuit)?; - if circuit.replica.source_endpoint.is_some() { - import_remote_from_cursor(root, &circuit, Some(&cursor)) - } else { - import_from_cursor(root, &circuit, Some(&cursor)) - } -} - -/// Bring an imported reader journal to the latest source frontier before use. -/// -/// Canonical journals and exact point-in-time forks are intentionally left -/// untouched. Imported replicas bootstrap when no cursor exists and otherwise -/// resume from their durable cursor. A short, bounded lease retry lets several -/// local agent clients start concurrently without weakening `BatPak`'s -/// single-owner contract for any physical store. -/// -/// # Errors -/// Returns a typed replication failure when topology, evidence, transport, or -/// store ownership cannot be resolved within the bounded retry window. -pub fn refresh_reader( - root: &Path, - workspace_id: Option<&str>, - journal_id: &str, -) -> Result, TexoError> { - let config = TexoRootConfig::load(&root.join(".texo/config.toml")).map_err(|error| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - error.to_string(), - ) - })?; - let (workspace, journal) = config - .resolve_journal(workspace_id, Some(journal_id)) - .map_err(|error| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - error.to_string(), - ) - })?; - if journal.role != JournalRole::Replica - || journal.replica_mode != Some(ReplicaMode::ImportedReadModel) - { - return Ok(None); - } - for attempt in 1..=READER_REFRESH_ATTEMPTS { - let result = if cursor_path(root, &workspace.workspace_id, journal_id).exists() { - follow_once(root, Some(&workspace.workspace_id), journal_id) - } else { - bootstrap(root, Some(&workspace.workspace_id), journal_id) - }; - match result { - Ok(report) => return Ok(Some(report)), - Err(TexoError::Replication { - kind: ReplicationFailureKind::Busy, - .. - }) if attempt < READER_REFRESH_ATTEMPTS => { - std::thread::sleep(READER_REFRESH_BACKOFF); - } - Err(error) => return Err(error), - } - } - Err(replication_error( - ReplicationFailureKind::Busy, - Committed::No, - "replica reader refresh exhausted its bounded lease retries", - )) -} - -fn bootstrap_exact(root: &Path, circuit: &Circuit) -> Result { - let source = open_store(&circuit.source_path, Committed::No)?; - let source_frontier = frontier(&source); - let report = substrate::exact_fork(&source, &circuit.replica_path).map_err(|error| { - replication_error( - ReplicationFailureKind::Substrate, - Committed::Unknown, - format!("exact fork failed: {error}"), - ) - })?; - let forked = open_read_only_store(&circuit.replica_path, Committed::Yes)?; - let events_verified = substrate::verify_intact(&forked).map_err(|error| { - replication_error( - ReplicationFailureKind::Verification, - Committed::Yes, - error.to_string(), - ) - })?; - let replica_frontier = frontier(&forked); - if source_frontier != replica_frontier { - return Err(replication_error( - ReplicationFailureKind::Verification, - Committed::Yes, - format!( - "fork frontier {replica_frontier} does not match source frontier {source_frontier}" - ), - )); - } - let evidence = ExactForkEvidence { - schema_version: STATE_SCHEMA_VERSION, - workspace_id: circuit.workspace.workspace_id.clone(), - source_journal: circuit.source.id.to_string(), - replica_journal: circuit.replica.id.to_string(), - fork_id_hex: hex_lower(&report.body.fork_id), - report_hash_hex: hex_lower(&report.body_hash), - source_frontier, - replica_frontier, - events_verified, - }; - let path = evidence_path( - root, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - "fork-evidence.msgpack", - ); - persist(&path, &evidence, Committed::Yes)?; - Ok(ReplicaReport::ExactFork { - evidence, - evidence_path: display_relative(root, &path), - }) -} - -fn bootstrap_imported(root: &Path, circuit: &Circuit) -> Result { - if circuit.replica.source_endpoint.is_some() { - import_remote_from_cursor(root, circuit, None) - } else { - import_from_cursor(root, circuit, None) - } -} - -fn import_from_cursor( - root: &Path, - circuit: &Circuit, - previous: Option<&ReplicaCursor>, -) -> Result { - let source = open_read_only_store(&circuit.source_path, Committed::No)?; - if let Some(cursor) = previous.as_ref() { - validate_source_anchor(&source, cursor)?; - } - let namespace = source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); - let mut after = previous - .as_ref() - .and_then(|cursor| cursor.source_high_watermark); - let source_ceiling = frontier(&source); - if let Some(cursor) = unchanged_local_cursor(previous, source_ceiling) { - return Ok(unchanged_report(root, circuit, cursor)); - } - let destination = open_store(&circuit.replica_path, Committed::No)?; - let represented = represented_source_events( - &destination, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - &namespace, - )?; - let mut imported = 0_u64; - let mut deduplicated = 0_u64; - let mut skipped_reserved = 0_u64; - let mut skipped_operational = 0_u64; - loop { - let page = substrate::read_import_page( - &source, - after, - source_ceiling, - &represented, - ReplicaBatchMaterializedV1::KIND, - ) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Substrate, - Committed::Unknown, - format!("source page read failed: {error}"), - ) - })?; - deduplicated = deduplicated.saturating_add(page.deduplicated); - skipped_reserved = skipped_reserved.saturating_add(page.skipped_reserved); - skipped_operational = skipped_operational.saturating_add(page.skipped_operational); - let next = page.high_watermark; - let has_more = page.has_more; - if !page.events.is_empty() { - let ledger = replica_ledger_item(circuit, &namespace, &page.events)?; - let count = - substrate::append_with_ledger(&destination, &namespace, page.events, ledger) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Substrate, - Committed::Unknown, - format!("atomic replica batch failed: {error}"), - ) - })?; - imported = imported.saturating_add(u64::try_from(count).unwrap_or(u64::MAX)); - } - if next == after || !has_more { - after = next; - break; - } - after = next; - } - verify_replica(&destination)?; - let source_high_watermark = after; - let source_anchor_event_id_hex = - source_high_watermark.and_then(|sequence| substrate::event_id_at(&source, sequence)); - if source_high_watermark.is_some() && source_anchor_event_id_hex.is_none() { - return Err(replication_error( - ReplicationFailureKind::AnchorMismatch, - Committed::Partial, - "source high watermark has no readable anchor event", - )); - } - let replica_frontier = frontier(&destination); - let cursor = ReplicaCursor { - schema_version: STATE_SCHEMA_VERSION, - workspace_id: circuit.workspace.workspace_id.clone(), - source_journal: circuit.source.id.to_string(), - replica_journal: circuit.replica.id.to_string(), - source_namespace: namespace, - source_endpoint: None, - source_high_watermark, - source_anchor_event_id_hex, - replica_frontier, - replica_anchor_event_id_hex: substrate::event_id_at(&destination, replica_frontier), - }; - let path = cursor_path( - root, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - ); - persist(&path, &cursor, Committed::Partial)?; - Ok(ReplicaReport::ImportedReadModel { - imported, - deduplicated, - skipped_reserved, - skipped_operational, - cursor, - cursor_path: display_relative(root, &path), - }) -} - -fn import_remote_from_cursor( - root: &Path, - circuit: &Circuit, - previous: Option<&ReplicaCursor>, -) -> Result { - let (endpoint, token) = remote_credentials(circuit)?; - let namespace = source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); - let mut progress = RemoteProgress { - after: previous.and_then(|cursor| cursor.source_high_watermark), - after_anchor: previous.and_then(|cursor| cursor.source_anchor_event_id_hex.clone()), - ..RemoteProgress::default() - }; - let first = request_remote_page(circuit, endpoint, &token, &progress)?; - validate_remote_response( - circuit, - progress.after, - None, - None, - &first, - progress.imported, - )?; - if let Some(cursor) = previous.filter(|_| { - first.high_watermark == progress.after - && !first.has_more - && first.events.is_empty() - && first.source_ceiling == progress.after.unwrap_or(0) - }) { - return Ok(unchanged_report(root, circuit, cursor)); - } - let destination = open_store(&circuit.replica_path, Committed::No)?; - let mut represented = represented_source_events( - &destination, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - &namespace, - )?; - let mut next_response = Some(first); - - loop { - let response = if let Some(first) = next_response.take() { - first - } else { - request_remote_page(circuit, endpoint, &token, &progress)? - }; - let has_more = response.has_more; - materialize_remote_page( - circuit, - &destination, - &namespace, - &mut represented, - &mut progress, - response, - )?; - if !has_more { - break; - } - } - - finish_remote_import(root, circuit, endpoint, &destination, namespace, progress) -} - -fn remote_credentials(circuit: &Circuit) -> Result<(&str, String), TexoError> { - let endpoint = circuit.replica.source_endpoint.as_deref().ok_or_else(|| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - "remote replica has no source endpoint", - ) - })?; - let token_env = circuit.replica.source_token_env.as_deref().ok_or_else(|| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - "remote replica has no token environment variable", - ) - })?; - let token = std::env::var(token_env).map_err(|_| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - format!("remote replica token environment variable `{token_env}` is not set"), - ) - })?; - if token.is_empty() { - Err(replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - format!("remote replica token environment variable `{token_env}` is empty"), - )) - } else { - Ok((endpoint, token)) - } -} - -fn request_remote_page( - circuit: &Circuit, - endpoint: &str, - token: &str, - progress: &RemoteProgress, -) -> Result { - let request = PageRequest::authenticated( - token, - circuit.workspace.workspace_id.clone(), - circuit.source.id.to_string(), - progress.after, - progress.after_anchor.clone(), - progress.source_ceiling, - ) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Evidence, - committed_after(progress.imported), - format!("authenticate remote replica page request: {error}"), - ) - })?; - let input = batpak::canonical::to_bytes(&request).map_err(|error| { - replication_error( - ReplicationFailureKind::Evidence, - committed_after(progress.imported), - format!("encode remote replica page request: {error}"), - ) - })?; - let response = crate::compat::netbat::call( - endpoint, - replica_net::PAGE_OPERATION, - &input, - &replica_net::limits(), - replica_net::REQUEST_TIMEOUT, - ) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Transport, - committed_after(progress.imported), - format!("remote replica page call: {error}"), - ) - })?; - batpak::canonical::from_bytes(&response.into_bytes()).map_err(|error| { - replication_error( - ReplicationFailureKind::Transport, - committed_after(progress.imported), - format!("decode remote replica page: {error}"), - ) - }) -} - -fn materialize_remote_page( - circuit: &Circuit, - destination: &Store, - namespace: &str, - represented: &mut BTreeSet, - progress: &mut RemoteProgress, - response: PageResponse, -) -> Result<(), TexoError> { - validate_remote_response( - circuit, - progress.after, - progress.source_ceiling, - progress.source_ceiling_anchor.as_deref(), - &response, - progress.imported, - )?; - let next = response.high_watermark; - if next == progress.after && response.has_more { - return Err(replication_error( - ReplicationFailureKind::Verification, - committed_after(progress.imported), - "remote replica page made no cursor progress", - )); - } - let events = validate_remote_events(&response, progress, represented)?; - if !events.is_empty() { - let ledger = replica_ledger_item(circuit, namespace, &events)?; - let count = substrate::append_with_ledger(destination, namespace, events, ledger).map_err( - |error| { - replication_error( - ReplicationFailureKind::Substrate, - committed_after(progress.imported), - format!("atomic remote replica batch failed: {error}"), - ) - }, - )?; - progress.imported = progress - .imported - .saturating_add(u64::try_from(count).unwrap_or(u64::MAX)); - } - progress.source_ceiling = Some(response.source_ceiling); - progress - .source_ceiling_anchor - .clone_from(&response.source_ceiling_anchor_event_id_hex); - progress.skipped_reserved = progress - .skipped_reserved - .saturating_add(response.skipped_reserved); - progress.skipped_operational = progress - .skipped_operational - .saturating_add(response.skipped_operational); - progress.after = next; - progress.after_anchor = response.high_watermark_anchor_event_id_hex; - Ok(()) -} - -fn validate_remote_events( - response: &PageResponse, - progress: &mut RemoteProgress, - represented: &mut BTreeSet, -) -> Result, TexoError> { - let mut events = Vec::with_capacity(response.events.len()); - let mut prior_sequence = progress.after; - for remote in response.events.iter().cloned() { - let sequence = remote.source.global_sequence; - if prior_sequence.is_some_and(|prior| sequence <= prior) - || sequence > response.source_ceiling - || response.high_watermark.is_none_or(|high| sequence > high) - { - return Err(replication_error( - ReplicationFailureKind::Verification, - committed_after(progress.imported), - "remote replica page event order is invalid", - )); - } - prior_sequence = Some(sequence); - if represented.contains(&remote.source.event_id_hex) { - progress.deduplicated = progress.deduplicated.saturating_add(1); - continue; - } - let event = substrate::ImportEvent::from_remote(remote).map_err(|error| { - replication_error( - ReplicationFailureKind::Verification, - committed_after(progress.imported), - format!("remote replica event failed validation: {error}"), - ) - })?; - represented.insert(event.source.event_id_hex.clone()); - events.push(event); - } - Ok(events) -} - -fn finish_remote_import( - root: &Path, - circuit: &Circuit, - endpoint: &str, - destination: &Store, - namespace: String, - progress: RemoteProgress, -) -> Result { - verify_replica(destination)?; - let replica_frontier = frontier(destination); - let cursor = ReplicaCursor { - schema_version: STATE_SCHEMA_VERSION, - workspace_id: circuit.workspace.workspace_id.clone(), - source_journal: circuit.source.id.to_string(), - replica_journal: circuit.replica.id.to_string(), - source_namespace: namespace, - source_endpoint: Some(endpoint.to_string()), - source_high_watermark: progress.after, - source_anchor_event_id_hex: progress.after_anchor, - replica_frontier, - replica_anchor_event_id_hex: substrate::event_id_at(destination, replica_frontier), - }; - let path = cursor_path( - root, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - ); - persist(&path, &cursor, committed_after(progress.imported))?; - Ok(ReplicaReport::ImportedReadModel { - imported: progress.imported, - deduplicated: progress.deduplicated, - skipped_reserved: progress.skipped_reserved, - skipped_operational: progress.skipped_operational, - cursor, - cursor_path: display_relative(root, &path), - }) -} - -fn unchanged_report(root: &Path, circuit: &Circuit, cursor: &ReplicaCursor) -> ReplicaReport { - let path = cursor_path( - root, - &circuit.workspace.workspace_id, - circuit.replica.id.as_str(), - ); - ReplicaReport::ImportedReadModel { - imported: 0, - deduplicated: 0, - skipped_reserved: 0, - skipped_operational: 0, - cursor: cursor.clone(), - cursor_path: display_relative(root, &path), - } -} - -fn unchanged_local_cursor( - previous: Option<&ReplicaCursor>, - source_ceiling: u64, -) -> Option<&ReplicaCursor> { - previous.filter(|cursor| { - cursor.source_high_watermark == Some(source_ceiling) - || (source_ceiling == 0 && cursor.source_high_watermark.is_none()) - }) -} - -fn validate_remote_response( - circuit: &Circuit, - after: Option, - expected_ceiling: Option, - expected_ceiling_anchor: Option<&str>, - response: &PageResponse, - imported: u64, -) -> Result<(), TexoError> { - let binding_matches = response.schema_version == 1 - && response.workspace_id == circuit.workspace.workspace_id - && response.source_journal == circuit.source.id.as_str(); - let ceiling_matches = expected_ceiling.is_none_or(|ceiling| response.source_ceiling == ceiling); - let ceiling_anchor_stable = expected_ceiling_anchor.is_none_or(|anchor| { - response.source_ceiling_anchor_event_id_hex.as_deref() == Some(anchor) - }); - let ceiling_anchor_matches = - response.source_ceiling == 0 || response.source_ceiling_anchor_event_id_hex.is_some(); - let cursor_valid = match response.high_watermark { - Some(high) => { - after.is_none_or(|previous| high >= previous) - && high <= response.source_ceiling - && response.high_watermark_anchor_event_id_hex.is_some() - } - None => after.is_none() && response.high_watermark_anchor_event_id_hex.is_none(), - }; - let completed_at_ceiling = response.has_more - || response.source_ceiling == 0 - || (response.high_watermark == Some(response.source_ceiling) - && response.high_watermark_anchor_event_id_hex - == response.source_ceiling_anchor_event_id_hex); - if binding_matches - && ceiling_matches - && ceiling_anchor_stable - && ceiling_anchor_matches - && cursor_valid - && completed_at_ceiling - { - Ok(()) - } else { - Err(replication_error( - ReplicationFailureKind::Verification, - committed_after(imported), - "remote replica response evidence is inconsistent", - )) - } -} - -const fn committed_after(imported: u64) -> Committed { - if imported == 0 { - Committed::No - } else { - Committed::Partial - } -} - -fn verify_replica(destination: &Store) -> Result<(), TexoError> { - substrate::verify_intact(destination) - .map(|_| ()) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Verification, - Committed::Partial, - error.to_string(), - ) - }) -} - -fn replica_ledger_item( - circuit: &Circuit, - namespace: &str, - events: &[substrate::ImportEvent], -) -> Result { - let entries = events - .iter() - .map(|event| ReplicaSourceEventV1 { - source_event_id_hex: event.source.event_id_hex.clone(), - source_global_sequence: event.source.global_sequence, - source_kind: event.source.kind_raw, - source_content_hash_hex: hex_lower(&event.source.content_hash), - }) - .collect::>(); - let first = entries.first().map_or_else( - || "0".to_string(), - |entry| entry.source_global_sequence.to_string(), - ); - let last = entries.last().map_or_else( - || "0".to_string(), - |entry| entry.source_global_sequence.to_string(), - ); - let payload = ReplicaBatchMaterializedV1 { - workspace_id: circuit.workspace.workspace_id.clone(), - source_journal: circuit.source.id.to_string(), - replica_journal: circuit.replica.id.to_string(), - source_namespace: namespace.to_string(), - events: entries, - }; - let coordinate = - replica_ledger_coordinate(&circuit.workspace.workspace_id, circuit.replica.id.as_str())?; - let options = AppendOptions::new().with_idempotency(IdempotencyKey::for_operation( - "texo.replica.batch.v1", - &[ - &circuit.workspace.workspace_id, - circuit.source.id.as_str(), - circuit.replica.id.as_str(), - &first, - &last, - ], - )); - BatchAppendItem::typed(coordinate, &payload, options, CausationRef::None).map_err(|error| { - replication_error( - ReplicationFailureKind::Substrate, - Committed::No, - format!("replica ledger encoding failed: {error}"), - ) - }) -} - -fn represented_source_events( - destination: &Store, - workspace: &str, - replica: &str, - namespace: &str, -) -> Result, TexoError> { - let coordinate = replica_ledger_coordinate(workspace, replica)?; - let mut represented = BTreeSet::new(); - for entry in destination.by_entity(coordinate.entity()) { - if entry.event_kind() != ReplicaBatchMaterializedV1::KIND { - continue; - } - let raw = destination.read_raw(entry.event_id()).map_err(|error| { - replication_error( - ReplicationFailureKind::Verification, - Committed::No, - format!("replica ledger read failed: {error}"), - ) - })?; - let payload: ReplicaBatchMaterializedV1 = batpak::encoding::from_bytes(&raw.event.payload) - .map_err(|error| { - replication_error( - ReplicationFailureKind::Verification, - Committed::No, - format!("replica ledger decode failed: {error}"), - ) - })?; - if payload.workspace_id != workspace - || payload.replica_journal != replica - || payload.source_namespace != namespace - { - return Err(replication_error( - ReplicationFailureKind::Verification, - Committed::No, - "replica ledger binding does not match the configured circuit", - )); - } - represented.extend( - payload - .events - .into_iter() - .map(|source| source.source_event_id_hex), - ); - } - Ok(represented) -} - -fn replica_ledger_coordinate(workspace: &str, replica: &str) -> Result { - Coordinate::new( - format!("replica-ledger:{workspace}:{replica}"), - format!("replication:{workspace}"), - ) - .map_err(TexoError::from) -} - -fn resolve_circuit( - root: &Path, - workspace_id: Option<&str>, - replica_id: &str, -) -> Result { - let config_path = root.join(".texo/config.toml"); - let root_config = TexoRootConfig::load(&config_path).map_err(|error| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - error.to_string(), - ) - })?; - let (workspace, replica) = root_config - .resolve_journal(workspace_id, Some(replica_id)) - .map_err(|error| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - error.to_string(), - ) - })?; - if replica.role != JournalRole::Replica { - return Err(replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - format!("journal `{replica_id}` is not a replica"), - )); - } - let source_id = replica.source_journal.as_ref().ok_or_else(|| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - "replica has no source journal", - ) - })?; - let (source_config, source) = root_config - .resolve_journal(Some(&workspace.workspace_id), Some(source_id.as_str())) - .map_err(|error| { - replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - error.to_string(), - ) - })?; - let source_path = source_config.store_path_buf(root); - let replica_path = workspace.store_path_buf(root); - if normalize_path(&source_path)? == normalize_path(&replica_path)? { - return Err(replication_error( - ReplicationFailureKind::InvalidTopology, - Committed::No, - "source and replica resolve to the same physical path", - )); - } - Ok(Circuit { - workspace, - source, - replica, - source_path, - replica_path, - }) -} - -fn ensure_fresh_destination(path: &Path) -> Result<(), TexoError> { - if !path.exists() { - return Ok(()); - } - let metadata = fs::symlink_metadata(path)?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(replication_error( - ReplicationFailureKind::DestinationNotFresh, - Committed::No, - format!("destination {} is not a fresh directory", path.display()), - )); - } - let mut entries = fs::read_dir(path)?; - if entries.next().transpose()?.is_some() { - return Err(replication_error( - ReplicationFailureKind::DestinationNotFresh, - Committed::No, - format!("destination {} is not empty", path.display()), - )); - } - Ok(()) -} - -fn open_store(path: &Path, committed: Committed) -> Result, TexoError> { - Store::open(StoreConfig::new(path)).map_err(|error| { - replication_error( - store_error_kind(&error), - committed, - format!("store {}: {error}", path.display()), - ) - }) -} - -fn open_read_only_store(path: &Path, committed: Committed) -> Result, TexoError> { - Store::::open_read_only(StoreConfig::new(path)).map_err(|error| { - replication_error( - store_error_kind(&error), - committed, - format!("read-only store {}: {error}", path.display()), - ) - }) -} - -fn store_error_kind(error: &batpak::store::StoreError) -> ReplicationFailureKind { - if matches!(error, batpak::store::StoreError::StoreLocked { .. }) { - ReplicationFailureKind::Busy - } else { - ReplicationFailureKind::Substrate - } -} - -fn validate_cursor_binding(cursor: &ReplicaCursor, circuit: &Circuit) -> Result<(), TexoError> { - let expected_namespace = - source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); - if cursor.schema_version != STATE_SCHEMA_VERSION - || cursor.workspace_id != circuit.workspace.workspace_id - || cursor.source_journal != circuit.source.id.as_str() - || cursor.replica_journal != circuit.replica.id.as_str() - || cursor.source_namespace != expected_namespace - || cursor.source_endpoint != circuit.replica.source_endpoint - { - return Err(replication_error( - ReplicationFailureKind::Evidence, - Committed::No, - "replica cursor does not match the configured circuit", - )); - } - Ok(()) -} - -fn validate_source_anchor( - source: &Store, - cursor: &ReplicaCursor, -) -> Result<(), TexoError> { - let Some(sequence) = cursor.source_high_watermark else { - return Ok(()); - }; - let actual = substrate::event_id_at(source, sequence); - if actual != cursor.source_anchor_event_id_hex { - return Err(replication_error( - ReplicationFailureKind::AnchorMismatch, - Committed::No, - format!("source anchor changed at global sequence {sequence}"), - )); - } - Ok(()) -} +mod evidence; +mod lifecycle; +mod materialize; +mod remote; -fn load_cursor(root: &Path, workspace: &str, replica: &str) -> Result { - let path = cursor_path(root, workspace, replica); - let bytes = fs::read(&path).map_err(|error| { - replication_error( - ReplicationFailureKind::Evidence, - Committed::No, - format!("read {}: {error}", path.display()), - ) - })?; - batpak::encoding::from_bytes(&bytes).map_err(|error| { - replication_error( - ReplicationFailureKind::Evidence, - Committed::No, - format!("decode {}: {error}", path.display()), - ) - }) -} - -fn persist(path: &Path, value: &T, committed: Committed) -> Result<(), TexoError> { - let bytes = batpak::encoding::to_bytes(value).map_err(|error| { - replication_error( - ReplicationFailureKind::Evidence, - committed, - format!("encode {}: {error}", path.display()), - ) - })?; - let parent = path.parent().ok_or_else(|| { - replication_error( - ReplicationFailureKind::Evidence, - committed, - "replication evidence path has no parent", - ) - })?; - fs::create_dir_all(parent)?; - let temporary = path.with_extension(format!("tmp-{}", std::process::id())); - let result = (|| -> Result<(), std::io::Error> { - let mut file = OpenOptions::new() - .create_new(true) - .write(true) - .open(&temporary)?; - file.write_all(&bytes)?; - file.sync_all()?; - fs::rename(&temporary, path)?; - if let Ok(directory) = OpenOptions::new().read(true).open(parent) { - directory.sync_all()?; - } - Ok(()) - })(); - if let Err(error) = result { - let _ignored = fs::remove_file(&temporary); - return Err(replication_error( - ReplicationFailureKind::Evidence, - committed, - format!("persist {}: {error}", path.display()), - )); - } - Ok(()) -} - -fn cursor_path(root: &Path, workspace: &str, replica: &str) -> PathBuf { - evidence_path(root, workspace, replica, "cursor.msgpack") -} - -fn evidence_path(root: &Path, workspace: &str, replica: &str, file: &str) -> PathBuf { - root.join(".texo") - .join("replication") - .join(workspace) - .join(replica) - .join(file) -} - -fn source_namespace(workspace: &str, source: &str) -> String { - format!("texo.replica.v1:{workspace}:{source}") -} - -fn frontier(store: &Store) -> u64 { - store.frontier().visible_hlc.global_sequence -} - -fn normalize_path(path: &Path) -> Result { - let absolute = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir()?.join(path) - }; - let mut normalized = PathBuf::new(); - for component in absolute.components() { - use std::path::Component; - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(component.as_os_str()), - Component::CurDir => {} - Component::ParentDir => { - let _ = normalized.pop(); - } - Component::Normal(value) => normalized.push(value), - } - } - Ok(normalized) -} - -fn display_relative(root: &Path, path: &Path) -> String { - path.strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .into_owned() -} - -fn hex_lower(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut output = String::with_capacity(bytes.len() * 2); - for byte in bytes { - output.push(char::from(HEX[usize::from(byte >> 4)])); - output.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - output -} - -fn replication_error( - kind: ReplicationFailureKind, - committed: Committed, - detail: impl Into, -) -> TexoError { - TexoError::Replication { - kind, - committed, - detail: detail.into(), - } -} +pub use lifecycle::{bootstrap, follow_once, refresh_reader}; diff --git a/src/replication/evidence.rs b/src/replication/evidence.rs new file mode 100644 index 0000000..21f3540 --- /dev/null +++ b/src/replication/evidence.rs @@ -0,0 +1,328 @@ +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use batpak::store::{Open, ReadOnly, Store, StoreConfig, StoreState}; +use serde::Serialize; + +use crate::compat::batpak as substrate; +use crate::config::TexoRootConfig; +use crate::error::{Committed, ReplicationFailureKind, TexoError}; +use crate::topology::JournalRole; + +use super::{Circuit, ReplicaCursor, ReplicaReport, STATE_SCHEMA_VERSION}; + +pub(super) fn unchanged_report( + root: &Path, + circuit: &Circuit, + cursor: &ReplicaCursor, +) -> ReplicaReport { + let path = cursor_path( + root, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + ); + ReplicaReport::ImportedReadModel { + imported: 0, + deduplicated: 0, + skipped_reserved: 0, + skipped_operational: 0, + cursor: cursor.clone(), + cursor_path: display_relative(root, &path), + } +} + +pub(super) fn resolve_circuit( + root: &Path, + workspace_id: Option<&str>, + replica_id: &str, +) -> Result { + let config_path = root.join(".texo/config.toml"); + let root_config = TexoRootConfig::load(&config_path).map_err(|error| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + error.to_string(), + ) + })?; + let (workspace, replica) = root_config + .resolve_journal(workspace_id, Some(replica_id)) + .map_err(|error| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + error.to_string(), + ) + })?; + if replica.role != JournalRole::Replica { + return Err(replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + format!("journal `{replica_id}` is not a replica"), + )); + } + let source_id = replica.source_journal.as_ref().ok_or_else(|| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + "replica has no source journal", + ) + })?; + let (source_config, source) = root_config + .resolve_journal(Some(&workspace.workspace_id), Some(source_id.as_str())) + .map_err(|error| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + error.to_string(), + ) + })?; + let source_path = source_config.store_path_buf(root); + let replica_path = workspace.store_path_buf(root); + if normalize_path(&source_path)? == normalize_path(&replica_path)? { + return Err(replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + "source and replica resolve to the same physical path", + )); + } + Ok(Circuit { + workspace, + source, + replica, + source_path, + replica_path, + }) +} + +pub(super) fn ensure_fresh_destination(path: &Path) -> Result<(), TexoError> { + if !path.exists() { + return Ok(()); + } + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(replication_error( + ReplicationFailureKind::DestinationNotFresh, + Committed::No, + format!("destination {} is not a fresh directory", path.display()), + )); + } + let mut entries = fs::read_dir(path)?; + if entries.next().transpose()?.is_some() { + return Err(replication_error( + ReplicationFailureKind::DestinationNotFresh, + Committed::No, + format!("destination {} is not empty", path.display()), + )); + } + Ok(()) +} + +pub(super) fn open_store(path: &Path, committed: Committed) -> Result, TexoError> { + Store::open(StoreConfig::new(path)).map_err(|error| { + replication_error( + store_error_kind(&error), + committed, + format!("store {}: {error}", path.display()), + ) + }) +} + +pub(super) fn open_read_only_store( + path: &Path, + committed: Committed, +) -> Result, TexoError> { + Store::::open_read_only(StoreConfig::new(path)).map_err(|error| { + replication_error( + store_error_kind(&error), + committed, + format!("read-only store {}: {error}", path.display()), + ) + }) +} + +pub(super) fn store_error_kind(error: &batpak::store::StoreError) -> ReplicationFailureKind { + if matches!(error, batpak::store::StoreError::StoreLocked { .. }) { + ReplicationFailureKind::Busy + } else { + ReplicationFailureKind::Substrate + } +} + +pub(super) fn validate_cursor_binding( + cursor: &ReplicaCursor, + circuit: &Circuit, +) -> Result<(), TexoError> { + let expected_namespace = + source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); + if cursor.schema_version != STATE_SCHEMA_VERSION + || cursor.workspace_id != circuit.workspace.workspace_id + || cursor.source_journal != circuit.source.id.as_str() + || cursor.replica_journal != circuit.replica.id.as_str() + || cursor.source_namespace != expected_namespace + || cursor.source_endpoint != circuit.replica.source_endpoint + { + return Err(replication_error( + ReplicationFailureKind::Evidence, + Committed::No, + "replica cursor does not match the configured circuit", + )); + } + Ok(()) +} + +pub(super) fn validate_source_anchor( + source: &Store, + cursor: &ReplicaCursor, +) -> Result<(), TexoError> { + let Some(sequence) = cursor.source_high_watermark else { + return Ok(()); + }; + let actual = substrate::event_id_at(source, sequence); + if actual != cursor.source_anchor_event_id_hex { + return Err(replication_error( + ReplicationFailureKind::AnchorMismatch, + Committed::No, + format!("source anchor changed at global sequence {sequence}"), + )); + } + Ok(()) +} + +pub(super) fn load_cursor( + root: &Path, + workspace: &str, + replica: &str, +) -> Result { + let path = cursor_path(root, workspace, replica); + let bytes = fs::read(&path).map_err(|error| { + replication_error( + ReplicationFailureKind::Evidence, + Committed::No, + format!("read {}: {error}", path.display()), + ) + })?; + batpak::encoding::from_bytes(&bytes).map_err(|error| { + replication_error( + ReplicationFailureKind::Evidence, + Committed::No, + format!("decode {}: {error}", path.display()), + ) + }) +} + +pub(super) fn persist( + path: &Path, + value: &T, + committed: Committed, +) -> Result<(), TexoError> { + let bytes = batpak::encoding::to_bytes(value).map_err(|error| { + replication_error( + ReplicationFailureKind::Evidence, + committed, + format!("encode {}: {error}", path.display()), + ) + })?; + let parent = path.parent().ok_or_else(|| { + replication_error( + ReplicationFailureKind::Evidence, + committed, + "replication evidence path has no parent", + ) + })?; + fs::create_dir_all(parent)?; + let temporary = path.with_extension(format!("tmp-{}", std::process::id())); + let result = (|| -> Result<(), std::io::Error> { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + fs::rename(&temporary, path)?; + if let Ok(directory) = OpenOptions::new().read(true).open(parent) { + directory.sync_all()?; + } + Ok(()) + })(); + if let Err(error) = result { + let _ignored = fs::remove_file(&temporary); + return Err(replication_error( + ReplicationFailureKind::Evidence, + committed, + format!("persist {}: {error}", path.display()), + )); + } + Ok(()) +} + +pub(super) fn cursor_path(root: &Path, workspace: &str, replica: &str) -> PathBuf { + evidence_path(root, workspace, replica, "cursor.msgpack") +} + +pub(super) fn evidence_path(root: &Path, workspace: &str, replica: &str, file: &str) -> PathBuf { + root.join(".texo") + .join("replication") + .join(workspace) + .join(replica) + .join(file) +} + +pub(super) fn source_namespace(workspace: &str, source: &str) -> String { + format!("texo.replica.v1:{workspace}:{source}") +} + +pub(super) fn frontier(store: &Store) -> u64 { + store.frontier().visible_hlc.global_sequence +} + +pub(super) fn normalize_path(path: &Path) -> Result { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + use std::path::Component; + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + let _ = normalized.pop(); + } + Component::Normal(value) => normalized.push(value), + } + } + Ok(normalized) +} + +pub(super) fn display_relative(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .into_owned() +} + +pub(super) fn hex_lower(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(char::from(HEX[usize::from(byte >> 4)])); + output.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + output +} + +pub(super) fn replication_error( + kind: ReplicationFailureKind, + committed: Committed, + detail: impl Into, +) -> TexoError { + TexoError::Replication { + kind, + committed, + detail: detail.into(), + } +} diff --git a/src/replication/lifecycle.rs b/src/replication/lifecycle.rs new file mode 100644 index 0000000..d2ee349 --- /dev/null +++ b/src/replication/lifecycle.rs @@ -0,0 +1,362 @@ +use std::collections::BTreeSet; +use std::path::Path; +use std::time::Duration; + +use batpak::event::EventPayload; +use batpak::store::{Open, ReadOnly, Store}; + +use crate::compat::batpak as substrate; +use crate::config::TexoRootConfig; +use crate::error::{Committed, ReplicationFailureKind, TexoError}; +use crate::events::payloads::ReplicaBatchMaterializedV1; +use crate::topology::{JournalRole, ReplicaMode}; + +use super::evidence::{ + cursor_path, display_relative, ensure_fresh_destination, evidence_path, frontier, hex_lower, + load_cursor, open_read_only_store, open_store, persist, replication_error, resolve_circuit, + source_namespace, unchanged_report, validate_cursor_binding, validate_source_anchor, +}; +use super::materialize::{replica_ledger_item, represented_source_events, verify_replica}; +use super::remote::import_remote_from_cursor; +use super::{Circuit, ExactForkEvidence, ReplicaCursor, ReplicaReport, STATE_SCHEMA_VERSION}; + +const READER_REFRESH_ATTEMPTS: u32 = 50; +const READER_REFRESH_BACKOFF: Duration = Duration::from_millis(40); + +#[derive(Default)] +struct LocalProgress { + after: Option, + imported: u64, + deduplicated: u64, + skipped_reserved: u64, + skipped_operational: u64, +} + +/// Bootstrap the configured replica according to its frozen mode. +/// +/// Exact forks preserve event identities at one point in time. Imported read +/// models receive destination-local event ids plus an atomic replica ledger. +/// Existing destination bytes are never cleared by Texo. +/// +/// # Errors +/// Fails closed on malformed topology, non-fresh destinations, `BatPak` errors, +/// verification failures, or evidence persistence failures. +pub fn bootstrap( + root: &Path, + workspace_id: Option<&str>, + replica_id: &str, +) -> Result { + let circuit = resolve_circuit(root, workspace_id, replica_id)?; + ensure_fresh_destination(&circuit.replica_path)?; + match circuit.replica.replica_mode { + Some(ReplicaMode::ExactFork) => bootstrap_exact(root, &circuit), + Some(ReplicaMode::ImportedReadModel) => bootstrap_imported(root, &circuit), + None => Err(replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + "replica declaration has no materialization mode", + )), + } +} + +/// Resume one configured imported read model from its verified source cursor. +/// +/// # Errors +/// Fails closed when the circuit is not an imported read model, its cursor is +/// missing/mismatched, the source anchor changed, or import/verification fails. +pub fn follow_once( + root: &Path, + workspace_id: Option<&str>, + replica_id: &str, +) -> Result { + let circuit = resolve_circuit(root, workspace_id, replica_id)?; + if circuit.replica.replica_mode != Some(ReplicaMode::ImportedReadModel) { + return Err(replication_error( + ReplicationFailureKind::ModeMismatch, + Committed::No, + "only imported_read_model replicas can follow a changing source", + )); + } + let cursor = load_cursor(root, &circuit.workspace.workspace_id, replica_id)?; + validate_cursor_binding(&cursor, &circuit)?; + if circuit.replica.source_endpoint.is_some() { + import_remote_from_cursor(root, &circuit, Some(&cursor)) + } else { + import_from_cursor(root, &circuit, Some(&cursor)) + } +} + +/// Bring an imported reader journal to the latest source frontier before use. +/// +/// Canonical journals and exact point-in-time forks are intentionally left +/// untouched. Imported replicas bootstrap when no cursor exists and otherwise +/// resume from their durable cursor. A short, bounded lease retry lets several +/// local agent clients start concurrently without weakening `BatPak`'s +/// single-owner contract for any physical store. +/// +/// # Errors +/// Returns a typed replication failure when topology, evidence, transport, or +/// store ownership cannot be resolved within the bounded retry window. +pub fn refresh_reader( + root: &Path, + workspace_id: Option<&str>, + journal_id: &str, +) -> Result, TexoError> { + let config = TexoRootConfig::load(&root.join(".texo/config.toml")).map_err(|error| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + error.to_string(), + ) + })?; + let (workspace, journal) = config + .resolve_journal(workspace_id, Some(journal_id)) + .map_err(|error| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + error.to_string(), + ) + })?; + if journal.role != JournalRole::Replica + || journal.replica_mode != Some(ReplicaMode::ImportedReadModel) + { + return Ok(None); + } + for attempt in 1..=READER_REFRESH_ATTEMPTS { + let result = if cursor_path(root, &workspace.workspace_id, journal_id).exists() { + follow_once(root, Some(&workspace.workspace_id), journal_id) + } else { + bootstrap(root, Some(&workspace.workspace_id), journal_id) + }; + match result { + Ok(report) => return Ok(Some(report)), + Err(TexoError::Replication { + kind: ReplicationFailureKind::Busy, + .. + }) if attempt < READER_REFRESH_ATTEMPTS => { + std::thread::sleep(READER_REFRESH_BACKOFF); + } + Err(error) => return Err(error), + } + } + Err(replication_error( + ReplicationFailureKind::Busy, + Committed::No, + "replica reader refresh exhausted its bounded lease retries", + )) +} + +fn bootstrap_exact(root: &Path, circuit: &Circuit) -> Result { + let source = open_store(&circuit.source_path, Committed::No)?; + let source_frontier = frontier(&source); + let report = substrate::exact_fork(&source, &circuit.replica_path).map_err(|error| { + replication_error( + ReplicationFailureKind::Substrate, + Committed::Unknown, + format!("exact fork failed: {error}"), + ) + })?; + let forked = open_read_only_store(&circuit.replica_path, Committed::Yes)?; + let events_verified = substrate::verify_intact(&forked).map_err(|error| { + replication_error( + ReplicationFailureKind::Verification, + Committed::Yes, + error.to_string(), + ) + })?; + let replica_frontier = frontier(&forked); + if source_frontier != replica_frontier { + return Err(replication_error( + ReplicationFailureKind::Verification, + Committed::Yes, + format!( + "fork frontier {replica_frontier} does not match source frontier {source_frontier}" + ), + )); + } + let evidence = ExactForkEvidence { + schema_version: STATE_SCHEMA_VERSION, + workspace_id: circuit.workspace.workspace_id.clone(), + source_journal: circuit.source.id.to_string(), + replica_journal: circuit.replica.id.to_string(), + fork_id_hex: hex_lower(&report.body.fork_id), + report_hash_hex: hex_lower(&report.body_hash), + source_frontier, + replica_frontier, + events_verified, + }; + let path = evidence_path( + root, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + "fork-evidence.msgpack", + ); + persist(&path, &evidence, Committed::Yes)?; + Ok(ReplicaReport::ExactFork { + evidence, + evidence_path: display_relative(root, &path), + }) +} + +fn bootstrap_imported(root: &Path, circuit: &Circuit) -> Result { + if circuit.replica.source_endpoint.is_some() { + import_remote_from_cursor(root, circuit, None) + } else { + import_from_cursor(root, circuit, None) + } +} + +fn import_from_cursor( + root: &Path, + circuit: &Circuit, + previous: Option<&ReplicaCursor>, +) -> Result { + let source = open_read_only_store(&circuit.source_path, Committed::No)?; + if let Some(cursor) = previous.as_ref() { + validate_source_anchor(&source, cursor)?; + } + let namespace = source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); + let after = previous + .as_ref() + .and_then(|cursor| cursor.source_high_watermark); + let source_ceiling = frontier(&source); + if let Some(cursor) = unchanged_local_cursor(previous, source_ceiling) { + return Ok(unchanged_report(root, circuit, cursor)); + } + let destination = open_store(&circuit.replica_path, Committed::No)?; + let represented = represented_source_events( + &destination, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + &namespace, + )?; + let progress = materialize_local_pages( + &source, + &destination, + circuit, + &namespace, + &represented, + source_ceiling, + after, + )?; + finish_local_import(root, circuit, &source, &destination, namespace, &progress) +} + +fn materialize_local_pages( + source: &Store, + destination: &Store, + circuit: &Circuit, + namespace: &str, + represented: &BTreeSet, + source_ceiling: u64, + after: Option, +) -> Result { + let mut progress = LocalProgress { + after, + ..LocalProgress::default() + }; + loop { + let page = substrate::read_import_page( + source, + progress.after, + source_ceiling, + represented, + ReplicaBatchMaterializedV1::KIND, + ) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Substrate, + Committed::Unknown, + format!("source page read failed: {error}"), + ) + })?; + progress.deduplicated = progress.deduplicated.saturating_add(page.deduplicated); + progress.skipped_reserved = progress + .skipped_reserved + .saturating_add(page.skipped_reserved); + progress.skipped_operational = progress + .skipped_operational + .saturating_add(page.skipped_operational); + let next = page.high_watermark; + let has_more = page.has_more; + if !page.events.is_empty() { + let ledger = replica_ledger_item(circuit, namespace, &page.events)?; + let count = substrate::append_with_ledger(destination, namespace, page.events, ledger) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Substrate, + Committed::Unknown, + format!("atomic replica batch failed: {error}"), + ) + })?; + progress.imported = progress + .imported + .saturating_add(u64::try_from(count).unwrap_or(u64::MAX)); + } + if next == progress.after || !has_more { + progress.after = next; + break; + } + progress.after = next; + } + Ok(progress) +} + +fn finish_local_import( + root: &Path, + circuit: &Circuit, + source: &Store, + destination: &Store, + namespace: String, + progress: &LocalProgress, +) -> Result { + verify_replica(destination)?; + let source_high_watermark = progress.after; + let source_anchor_event_id_hex = + source_high_watermark.and_then(|sequence| substrate::event_id_at(source, sequence)); + if source_high_watermark.is_some() && source_anchor_event_id_hex.is_none() { + return Err(replication_error( + ReplicationFailureKind::AnchorMismatch, + Committed::Partial, + "source high watermark has no readable anchor event", + )); + } + let replica_frontier = frontier(destination); + let cursor = ReplicaCursor { + schema_version: STATE_SCHEMA_VERSION, + workspace_id: circuit.workspace.workspace_id.clone(), + source_journal: circuit.source.id.to_string(), + replica_journal: circuit.replica.id.to_string(), + source_namespace: namespace, + source_endpoint: None, + source_high_watermark, + source_anchor_event_id_hex, + replica_frontier, + replica_anchor_event_id_hex: substrate::event_id_at(destination, replica_frontier), + }; + let path = cursor_path( + root, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + ); + persist(&path, &cursor, Committed::Partial)?; + Ok(ReplicaReport::ImportedReadModel { + imported: progress.imported, + deduplicated: progress.deduplicated, + skipped_reserved: progress.skipped_reserved, + skipped_operational: progress.skipped_operational, + cursor, + cursor_path: display_relative(root, &path), + }) +} + +fn unchanged_local_cursor( + previous: Option<&ReplicaCursor>, + source_ceiling: u64, +) -> Option<&ReplicaCursor> { + previous.filter(|cursor| { + cursor.source_high_watermark == Some(source_ceiling) + || (source_ceiling == 0 && cursor.source_high_watermark.is_none()) + }) +} diff --git a/src/replication/materialize.rs b/src/replication/materialize.rs new file mode 100644 index 0000000..2b08c8c --- /dev/null +++ b/src/replication/materialize.rs @@ -0,0 +1,133 @@ +use std::collections::BTreeSet; + +use batpak::coordinate::Coordinate; +use batpak::event::EventPayload; +use batpak::id::IdempotencyKey; +use batpak::store::{AppendOptions, BatchAppendItem, CausationRef, Open, Store}; + +use crate::compat::batpak as substrate; +use crate::error::{Committed, ReplicationFailureKind, TexoError}; +use crate::events::payloads::{ReplicaBatchMaterializedV1, ReplicaSourceEventV1}; + +use super::evidence::{hex_lower, replication_error}; +use super::Circuit; + +pub(super) fn verify_replica(destination: &Store) -> Result<(), TexoError> { + substrate::verify_intact(destination) + .map(|_| ()) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Verification, + Committed::Partial, + error.to_string(), + ) + }) +} + +pub(super) fn replica_ledger_item( + circuit: &Circuit, + namespace: &str, + events: &[substrate::ImportEvent], +) -> Result { + let entries = events + .iter() + .map(|event| ReplicaSourceEventV1 { + source_event_id_hex: event.source.event_id_hex.clone(), + source_global_sequence: event.source.global_sequence, + source_kind: event.source.kind_raw, + source_content_hash_hex: hex_lower(&event.source.content_hash), + }) + .collect::>(); + let first = entries.first().map_or_else( + || "0".to_string(), + |entry| entry.source_global_sequence.to_string(), + ); + let last = entries.last().map_or_else( + || "0".to_string(), + |entry| entry.source_global_sequence.to_string(), + ); + let payload = ReplicaBatchMaterializedV1 { + workspace_id: circuit.workspace.workspace_id.clone(), + source_journal: circuit.source.id.to_string(), + replica_journal: circuit.replica.id.to_string(), + source_namespace: namespace.to_string(), + events: entries, + }; + let coordinate = + replica_ledger_coordinate(&circuit.workspace.workspace_id, circuit.replica.id.as_str())?; + let options = AppendOptions::new().with_idempotency(IdempotencyKey::for_operation( + "texo.replica.batch.v1", + &[ + &circuit.workspace.workspace_id, + circuit.source.id.as_str(), + circuit.replica.id.as_str(), + &first, + &last, + ], + )); + BatchAppendItem::typed(coordinate, &payload, options, CausationRef::None).map_err(|error| { + replication_error( + ReplicationFailureKind::Substrate, + Committed::No, + format!("replica ledger encoding failed: {error}"), + ) + }) +} + +pub(super) fn represented_source_events( + destination: &Store, + workspace: &str, + replica: &str, + namespace: &str, +) -> Result, TexoError> { + let coordinate = replica_ledger_coordinate(workspace, replica)?; + let mut represented = BTreeSet::new(); + for entry in destination.by_entity(coordinate.entity()) { + if entry.event_kind() != ReplicaBatchMaterializedV1::KIND { + continue; + } + let raw = destination.read_raw(entry.event_id()).map_err(|error| { + replication_error( + ReplicationFailureKind::Verification, + Committed::No, + format!("replica ledger read failed: {error}"), + ) + })?; + let payload: ReplicaBatchMaterializedV1 = batpak::encoding::from_bytes(&raw.event.payload) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Verification, + Committed::No, + format!("replica ledger decode failed: {error}"), + ) + })?; + if payload.workspace_id != workspace + || payload.replica_journal != replica + || payload.source_namespace != namespace + { + return Err(replication_error( + ReplicationFailureKind::Verification, + Committed::No, + "replica ledger binding does not match the configured circuit", + )); + } + represented.extend( + payload + .events + .into_iter() + .map(|source| source.source_event_id_hex), + ); + } + Ok(represented) +} + +pub(super) fn replica_ledger_coordinate( + workspace: &str, + replica: &str, +) -> Result { + Coordinate::new( + format!("replica-ledger:{workspace}:{replica}"), + format!("replication:{workspace}"), + ) + .map_err(TexoError::from) +} diff --git a/src/replication/remote.rs b/src/replication/remote.rs new file mode 100644 index 0000000..868d40a --- /dev/null +++ b/src/replication/remote.rs @@ -0,0 +1,357 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use batpak::store::{Open, Store}; + +use crate::compat::batpak as substrate; +use crate::error::{Committed, ReplicationFailureKind, TexoError}; +use crate::replica_net::{self, PageRequest, PageResponse}; + +use super::evidence::{ + cursor_path, display_relative, frontier, open_store, persist, replication_error, + source_namespace, unchanged_report, +}; +use super::materialize::{replica_ledger_item, represented_source_events, verify_replica}; +use super::{Circuit, ReplicaCursor, ReplicaReport, STATE_SCHEMA_VERSION}; + +#[derive(Default)] +struct RemoteProgress { + after: Option, + after_anchor: Option, + source_ceiling: Option, + source_ceiling_anchor: Option, + imported: u64, + deduplicated: u64, + skipped_reserved: u64, + skipped_operational: u64, +} + +pub(super) fn import_remote_from_cursor( + root: &Path, + circuit: &Circuit, + previous: Option<&ReplicaCursor>, +) -> Result { + let (endpoint, token) = remote_credentials(circuit)?; + let namespace = source_namespace(&circuit.workspace.workspace_id, circuit.source.id.as_str()); + let mut progress = RemoteProgress { + after: previous.and_then(|cursor| cursor.source_high_watermark), + after_anchor: previous.and_then(|cursor| cursor.source_anchor_event_id_hex.clone()), + ..RemoteProgress::default() + }; + let first = request_remote_page(circuit, endpoint, &token, &progress)?; + validate_remote_response( + circuit, + progress.after, + None, + None, + &first, + progress.imported, + )?; + if let Some(cursor) = previous.filter(|_| { + first.high_watermark == progress.after + && !first.has_more + && first.events.is_empty() + && first.source_ceiling == progress.after.unwrap_or(0) + }) { + return Ok(unchanged_report(root, circuit, cursor)); + } + let destination = open_store(&circuit.replica_path, Committed::No)?; + let mut represented = represented_source_events( + &destination, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + &namespace, + )?; + let mut next_response = Some(first); + + loop { + let response = if let Some(first) = next_response.take() { + first + } else { + request_remote_page(circuit, endpoint, &token, &progress)? + }; + let has_more = response.has_more; + materialize_remote_page( + circuit, + &destination, + &namespace, + &mut represented, + &mut progress, + response, + )?; + if !has_more { + break; + } + } + + finish_remote_import(root, circuit, endpoint, &destination, namespace, progress) +} + +fn remote_credentials(circuit: &Circuit) -> Result<(&str, String), TexoError> { + let endpoint = circuit.replica.source_endpoint.as_deref().ok_or_else(|| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + "remote replica has no source endpoint", + ) + })?; + let token_env = circuit.replica.source_token_env.as_deref().ok_or_else(|| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + "remote replica has no token environment variable", + ) + })?; + let token = std::env::var(token_env).map_err(|_| { + replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + format!("remote replica token environment variable `{token_env}` is not set"), + ) + })?; + if token.is_empty() { + Err(replication_error( + ReplicationFailureKind::InvalidTopology, + Committed::No, + format!("remote replica token environment variable `{token_env}` is empty"), + )) + } else { + Ok((endpoint, token)) + } +} + +fn request_remote_page( + circuit: &Circuit, + endpoint: &str, + token: &str, + progress: &RemoteProgress, +) -> Result { + let request = PageRequest::authenticated( + token, + circuit.workspace.workspace_id.clone(), + circuit.source.id.to_string(), + progress.after, + progress.after_anchor.clone(), + progress.source_ceiling, + ) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Evidence, + committed_after(progress.imported), + format!("authenticate remote replica page request: {error}"), + ) + })?; + let input = batpak::canonical::to_bytes(&request).map_err(|error| { + replication_error( + ReplicationFailureKind::Evidence, + committed_after(progress.imported), + format!("encode remote replica page request: {error}"), + ) + })?; + let response = crate::compat::netbat::call( + endpoint, + replica_net::PAGE_OPERATION, + &input, + &replica_net::limits(), + replica_net::REQUEST_TIMEOUT, + ) + .map_err(|error| { + replication_error( + ReplicationFailureKind::Transport, + committed_after(progress.imported), + format!("remote replica page call: {error}"), + ) + })?; + batpak::canonical::from_bytes(&response.into_bytes()).map_err(|error| { + replication_error( + ReplicationFailureKind::Transport, + committed_after(progress.imported), + format!("decode remote replica page: {error}"), + ) + }) +} + +fn materialize_remote_page( + circuit: &Circuit, + destination: &Store, + namespace: &str, + represented: &mut BTreeSet, + progress: &mut RemoteProgress, + response: PageResponse, +) -> Result<(), TexoError> { + validate_remote_response( + circuit, + progress.after, + progress.source_ceiling, + progress.source_ceiling_anchor.as_deref(), + &response, + progress.imported, + )?; + let next = response.high_watermark; + if next == progress.after && response.has_more { + return Err(replication_error( + ReplicationFailureKind::Verification, + committed_after(progress.imported), + "remote replica page made no cursor progress", + )); + } + let events = validate_remote_events(&response, progress, represented)?; + if !events.is_empty() { + let ledger = replica_ledger_item(circuit, namespace, &events)?; + let count = substrate::append_with_ledger(destination, namespace, events, ledger).map_err( + |error| { + replication_error( + ReplicationFailureKind::Substrate, + committed_after(progress.imported), + format!("atomic remote replica batch failed: {error}"), + ) + }, + )?; + progress.imported = progress + .imported + .saturating_add(u64::try_from(count).unwrap_or(u64::MAX)); + } + progress.source_ceiling = Some(response.source_ceiling); + progress + .source_ceiling_anchor + .clone_from(&response.source_ceiling_anchor_event_id_hex); + progress.skipped_reserved = progress + .skipped_reserved + .saturating_add(response.skipped_reserved); + progress.skipped_operational = progress + .skipped_operational + .saturating_add(response.skipped_operational); + progress.after = next; + progress.after_anchor = response.high_watermark_anchor_event_id_hex; + Ok(()) +} + +fn validate_remote_events( + response: &PageResponse, + progress: &mut RemoteProgress, + represented: &mut BTreeSet, +) -> Result, TexoError> { + let mut events = Vec::with_capacity(response.events.len()); + let mut prior_sequence = progress.after; + for remote in response.events.iter().cloned() { + let sequence = remote.source.global_sequence; + if prior_sequence.is_some_and(|prior| sequence <= prior) + || sequence > response.source_ceiling + || response.high_watermark.is_none_or(|high| sequence > high) + { + return Err(replication_error( + ReplicationFailureKind::Verification, + committed_after(progress.imported), + "remote replica page event order is invalid", + )); + } + prior_sequence = Some(sequence); + if represented.contains(&remote.source.event_id_hex) { + progress.deduplicated = progress.deduplicated.saturating_add(1); + continue; + } + let event = substrate::ImportEvent::from_remote(remote).map_err(|error| { + replication_error( + ReplicationFailureKind::Verification, + committed_after(progress.imported), + format!("remote replica event failed validation: {error}"), + ) + })?; + represented.insert(event.source.event_id_hex.clone()); + events.push(event); + } + Ok(events) +} + +fn finish_remote_import( + root: &Path, + circuit: &Circuit, + endpoint: &str, + destination: &Store, + namespace: String, + progress: RemoteProgress, +) -> Result { + verify_replica(destination)?; + let replica_frontier = frontier(destination); + let cursor = ReplicaCursor { + schema_version: STATE_SCHEMA_VERSION, + workspace_id: circuit.workspace.workspace_id.clone(), + source_journal: circuit.source.id.to_string(), + replica_journal: circuit.replica.id.to_string(), + source_namespace: namespace, + source_endpoint: Some(endpoint.to_string()), + source_high_watermark: progress.after, + source_anchor_event_id_hex: progress.after_anchor, + replica_frontier, + replica_anchor_event_id_hex: substrate::event_id_at(destination, replica_frontier), + }; + let path = cursor_path( + root, + &circuit.workspace.workspace_id, + circuit.replica.id.as_str(), + ); + persist(&path, &cursor, committed_after(progress.imported))?; + Ok(ReplicaReport::ImportedReadModel { + imported: progress.imported, + deduplicated: progress.deduplicated, + skipped_reserved: progress.skipped_reserved, + skipped_operational: progress.skipped_operational, + cursor, + cursor_path: display_relative(root, &path), + }) +} + +fn validate_remote_response( + circuit: &Circuit, + after: Option, + expected_ceiling: Option, + expected_ceiling_anchor: Option<&str>, + response: &PageResponse, + imported: u64, +) -> Result<(), TexoError> { + let binding_matches = response.schema_version == 1 + && response.workspace_id == circuit.workspace.workspace_id + && response.source_journal == circuit.source.id.as_str(); + let ceiling_matches = expected_ceiling.is_none_or(|ceiling| response.source_ceiling == ceiling); + let ceiling_anchor_stable = expected_ceiling_anchor.is_none_or(|anchor| { + response.source_ceiling_anchor_event_id_hex.as_deref() == Some(anchor) + }); + let ceiling_anchor_matches = + response.source_ceiling == 0 || response.source_ceiling_anchor_event_id_hex.is_some(); + let cursor_valid = match response.high_watermark { + Some(high) => { + after.is_none_or(|previous| high >= previous) + && high <= response.source_ceiling + && response.high_watermark_anchor_event_id_hex.is_some() + } + None => after.is_none() && response.high_watermark_anchor_event_id_hex.is_none(), + }; + let completed_at_ceiling = response.has_more + || response.source_ceiling == 0 + || (response.high_watermark == Some(response.source_ceiling) + && response.high_watermark_anchor_event_id_hex + == response.source_ceiling_anchor_event_id_hex); + if binding_matches + && ceiling_matches + && ceiling_anchor_stable + && ceiling_anchor_matches + && cursor_valid + && completed_at_ceiling + { + Ok(()) + } else { + Err(replication_error( + ReplicationFailureKind::Verification, + committed_after(imported), + "remote replica response evidence is inconsistent", + )) + } +} + +const fn committed_after(imported: u64) -> Committed { + if imported == 0 { + Committed::No + } else { + Committed::Partial + } +} diff --git a/src/semantics/mod.rs b/src/semantics/mod.rs index 4587e13..1e23ff2 100644 --- a/src/semantics/mod.rs +++ b/src/semantics/mod.rs @@ -6,7 +6,9 @@ pub mod chat; /// Hosted OpenRouter semantic backends. #[cfg(feature = "openrouter")] pub mod openrouter; +/// Semantic claim relation pipeline. pub mod pipeline; +pub(crate) mod score; pub mod traits; pub use traits::*; diff --git a/src/semantics/openrouter.rs b/src/semantics/openrouter.rs index 758af3f..1641933 100644 --- a/src/semantics/openrouter.rs +++ b/src/semantics/openrouter.rs @@ -688,320 +688,4 @@ fn parse_propose_response( } #[cfg(test)] -mod tests { - use super::*; - - fn test_role(role: ModelRole, model: &str) -> ResolvedRole { - resolve_role( - role, - &RoleOverrides { - api_key: Some("test-key".to_string()), - model: Some(model.to_string()), - ..RoleOverrides::default() - }, - None, - ) - } - - // --- request-body construction --- - - #[test] - fn embeddings_request_has_model_and_input_array() { - let body = build_embeddings_request("google/gemini-embedding-2", &["hello", "world"]); - assert_eq!(body["model"], "google/gemini-embedding-2"); - assert_eq!(body["input"], json!(["hello", "world"])); - } - - #[test] - fn default_provider_fingerprints_remain_byte_stable() { - let relate_role = test_role(ModelRole::Relate, "relation/model"); - let relate_client = OpenRouterClient::from_role(&relate_role).expect("client"); - let relater = OpenRouterRelater { - client: relate_client, - role: relate_role, - }; - assert_eq!( - relater.fingerprint(), - "openrouter:relation/model|relation-v2" - ); - - let propose_role = test_role(ModelRole::Propose, "propose/model"); - let propose_client = OpenRouterClient::from_role(&propose_role).expect("client"); - let proposer = OpenRouterProposer { - client: propose_client, - role: propose_role, - }; - assert_eq!( - proposer.fingerprint(), - "openrouter:propose/model|propose-v3" - ); - } - - // --- embeddings parsing --- - - #[test] - fn embeddings_response_parses_in_request_order() { - let value = json!({ - "data": [ - { "index": 1, "embedding": [0.1, 0.2] }, - { "index": 0, "embedding": [0.3, 0.4] } - ] - }); - let vectors = parse_embeddings_response(&value, 2).expect("parse"); - // Sorted by `index`: row 0 first. - assert_eq!(vectors, vec![vec![0.3, 0.4], vec![0.1, 0.2]]); - } - - #[test] - fn embeddings_response_without_index_keeps_array_order() { - let value = json!({ - "data": [ - { "embedding": [1.0] }, - { "embedding": [2.0] } - ] - }); - let vectors = parse_embeddings_response(&value, 2).expect("parse"); - assert_eq!(vectors, vec![vec![1.0], vec![2.0]]); - } - - #[test] - fn embeddings_response_wrong_count_is_error() { - let value = json!({ "data": [ { "embedding": [1.0] } ] }); - let err = parse_embeddings_response(&value, 2).expect_err("count mismatch"); - assert!(matches!(err, BackendError::UnexpectedResponse { .. })); - } - - #[test] - fn embeddings_response_empty_vector_is_error() { - let value = json!({ "data": [ { "embedding": [] } ] }); - let err = parse_embeddings_response(&value, 1).expect_err("empty vector"); - assert!(matches!(err, BackendError::UnexpectedResponse { .. })); - } - - #[test] - fn embeddings_response_bad_shape_is_parse_error() { - let value = json!({ "data": [ { "embedding": "not-a-vector" } ] }); - let err = parse_embeddings_response(&value, 1).expect_err("parse error"); - assert!(matches!(err, BackendError::Parse { .. })); - } - - /// Helper that wraps a judge content string in a chat-completions envelope. - fn chat_envelope(content: &str) -> Value { - json!({ "choices": [ { "message": { "role": "assistant", "content": content } } ] }) - } - - fn chat_envelope_with_reason(content: &str, finish_reason: &str) -> Value { - json!({ - "choices": [{ - "finish_reason": finish_reason, - "message": { "role": "assistant", "content": content } - }] - }) - } - - #[test] - fn positive_length_finish_is_truncated_but_unknown_reason_is_preserved() { - let truncated = parse_relation_response(&chat_envelope_with_reason("", "length"), 4096) - .expect_err("positive truncation"); - assert!(matches!( - truncated, - BackendError::Truncated { - finish_reason, - max_tokens: 4096, - .. - } if finish_reason == "length" - )); - - let unknown = - parse_relation_response(&chat_envelope_with_reason("", "weird_variant"), 4096) - .expect_err("unknown finish reason is malformed, not truncation"); - assert!(matches!(unknown, BackendError::UnexpectedResponse { .. })); - assert!(unknown.to_string().contains("weird_variant")); - } - // --- claim-relation parsing + label mapping --- - - #[test] - fn relation_request_carries_prompt_and_strict_format() { - let role = test_role(ModelRole::Relate, "nvidia/nemotron-3-ultra-550b-a55b"); - let body = build_relation_request(&role, "older X", "newer Y"); - assert_eq!(body["model"], "nvidia/nemotron-3-ultra-550b-a55b"); - assert_eq!(body["temperature"], json!(0.0)); - assert_eq!( - body["response_format"]["json_schema"]["name"], - "claim_relation" - ); - assert_eq!(body["max_tokens"], 4096); - let messages = body["messages"].as_array().expect("messages array"); - assert_eq!(messages.len(), 2); - let user = messages[1]["content"].as_str().expect("user content"); - assert!(user.contains("older X")); - assert!(user.contains("newer Y")); - // Order must be conveyed: older is labeled first, newer second. - assert!(user.find("older X") < user.find("newer Y")); - } - - #[test] - fn provider_capability_suppresses_unsupported_response_format() { - let mut role = test_role(ModelRole::Relate, "provider/model"); - role.profile.strict_json_schema_ok = false; - let body = build_relation_request(&role, "older", "newer"); - assert!(body.get("response_format").is_none()); - } - - #[test] - fn relation_label_strings_map_to_claim_relation() { - assert_eq!( - parse_relation_label("SUPERSEDES"), - Some(ClaimRelation::Supersedes) - ); - assert_eq!( - parse_relation_label(" conflict "), - Some(ClaimRelation::Conflict) - ); - assert_eq!( - parse_relation_label("duplicate"), - Some(ClaimRelation::Duplicate) - ); - assert_eq!( - parse_relation_label("unrelated"), - Some(ClaimRelation::Unrelated) - ); - assert_eq!(parse_relation_label("nonsense"), None); - } - - #[test] - fn relation_response_parses_clean_json() { - let value = chat_envelope("{\"relation\": \"supersedes\", \"score\": 0.88}"); - let verdict = parse_relation_response(&value, 4096).expect("parse"); - assert_eq!(verdict.relation, ClaimRelation::Supersedes); - assert!((verdict.score - 0.88).abs() < 1e-6); - } - - #[test] - fn relation_response_tolerates_fences_and_clamps_score() { - let value = chat_envelope("```json\n{\"relation\": \"conflict\", \"score\": 1.4}\n```"); - let verdict = parse_relation_response(&value, 4096).expect("parse"); - assert_eq!(verdict.relation, ClaimRelation::Conflict); - assert!((verdict.score - 1.0).abs() < 1e-6); - } - - #[test] - fn relation_response_unknown_label_is_error() { - let value = chat_envelope("{\"relation\": \"maybe\", \"score\": 0.5}"); - let err = parse_relation_response(&value, 4096).expect_err("unknown relation"); - assert!(matches!(err, BackendError::UnexpectedResponse { .. })); - } - - #[test] - fn relation_response_missing_score_is_parse_error() { - let value = chat_envelope("{\"relation\": \"unrelated\"}"); - let err = parse_relation_response(&value, 4096).expect_err("missing score"); - assert!(matches!(err, BackendError::Parse { .. })); - } - - // --- proposer parsing --- - - #[test] - fn propose_request_includes_heading_context_and_schema() { - let role = test_role(ModelRole::Propose, "anthropic/claude-opus-4.8"); - let body = build_propose_request( - &role, - "Deploys moved to Tuesday.", - &["Operations".to_owned(), "Deploys".to_owned()], - ); - assert_eq!(body["max_tokens"], 2048); - let user = body["messages"][1]["content"].as_str().expect("user"); - assert!(user.contains("Operations > Deploys"), "heading path joined"); - assert!(user.contains("Deploys moved to Tuesday.")); - } - - #[test] - fn propose_request_without_headings_has_no_section_line() { - let role = test_role(ModelRole::Propose, "m"); - let body = build_propose_request(&role, "Some span.", &[]); - let user = body["messages"][1]["content"].as_str().expect("user"); - assert!(!user.contains("Section:")); - assert!(user.starts_with("Span:")); - } - - #[test] - fn propose_response_parses_claims_and_rescales_confidence() { - let value = chat_envelope( - "{\"claims\":[{\"text\":\"Deploys moved to Tuesday.\",\"subject\":\"deploys\",\"predicate\":\"scheduled\",\"object\":\"Tuesday\",\"confidence\":90}]}", - ); - let claims = parse_propose_response(&value, 2048).expect("parse"); - assert_eq!(claims.len(), 1); - assert_eq!(claims[0].text, "Deploys moved to Tuesday."); - assert_eq!(claims[0].subject, "deploys"); - assert_eq!(claims[0].confidence_ppm, 900_000); - } - - #[test] - fn propose_response_drops_blank_text_and_clamps_confidence() { - let value = chat_envelope( - "{\"claims\":[{\"text\":\" \",\"subject\":\"\",\"predicate\":\"\",\"object\":\"\",\"confidence\":50},{\"text\":\"Real claim.\",\"subject\":\"x\",\"predicate\":\"y\",\"object\":\"z\",\"confidence\":170}]}", - ); - let claims = parse_propose_response(&value, 2048).expect("parse"); - assert_eq!(claims.len(), 1, "blank-text claim dropped"); - assert_eq!(claims[0].text, "Real claim."); - assert_eq!( - claims[0].confidence_ppm, 1_000_000, - "over-100 confidence clamped" - ); - } - - #[test] - fn propose_response_empty_list_is_ok() { - let value = chat_envelope("{\"claims\":[]}"); - let claims = parse_propose_response(&value, 2048).expect("parse"); - assert!(claims.is_empty()); - } - - #[test] - fn propose_response_tolerates_fences() { - let value = chat_envelope( - "```json\n{\"claims\":[{\"text\":\"A.\",\"subject\":\"a\",\"predicate\":\"b\",\"object\":\"c\",\"confidence\":40}]}\n```", - ); - let claims = parse_propose_response(&value, 2048).expect("parse"); - assert_eq!(claims.len(), 1); - assert_eq!(claims[0].confidence_ppm, 400_000); - } - - #[test] - fn propose_response_missing_claims_key_is_parse_error() { - let value = chat_envelope("{\"items\":[]}"); - let err = parse_propose_response(&value, 2048).expect_err("missing claims"); - assert!(matches!(err, BackendError::Parse { .. })); - } - - /// PROVES: the chat-completions response parsers are TOTAL functions on - /// arbitrary model output — they return `Ok` or a typed `Err`, never panic — - /// and when `Ok`, scores are clamped to `[0,1]` and confidence to `0..=1e6`. - /// The cargo-fuzz substitute for the judge/proposer parsers. - mod robustness { - use super::super::{parse_propose_response, parse_relation_response}; - use super::chat_envelope; - use proptest::prelude::*; - - proptest! { - #![proptest_config(ProptestConfig::with_cases(128))] - - #[test] - fn relation_parser_is_total(content in any::()) { - if let Ok(v) = parse_relation_response(&chat_envelope(&content), 4096) { - prop_assert!((0.0..=1.0).contains(&v.score)); - } - } - - #[test] - fn propose_parser_is_total(content in any::()) { - if let Ok(claims) = parse_propose_response(&chat_envelope(&content), 2048) { - for c in claims { - prop_assert!(c.confidence_ppm <= 1_000_000); - prop_assert!(!c.text.trim().is_empty()); - } - } - } - } - } -} +mod tests; diff --git a/src/semantics/openrouter/tests.rs b/src/semantics/openrouter/tests.rs new file mode 100644 index 0000000..8e2dd52 --- /dev/null +++ b/src/semantics/openrouter/tests.rs @@ -0,0 +1,314 @@ +use super::*; + +fn test_role(role: ModelRole, model: &str) -> ResolvedRole { + resolve_role( + role, + &RoleOverrides { + api_key: Some("test-key".to_string()), + model: Some(model.to_string()), + ..RoleOverrides::default() + }, + None, + ) +} + +// --- request-body construction --- + +#[test] +fn embeddings_request_has_model_and_input_array() { + let body = build_embeddings_request("google/gemini-embedding-2", &["hello", "world"]); + assert_eq!(body["model"], "google/gemini-embedding-2"); + assert_eq!(body["input"], json!(["hello", "world"])); +} + +#[test] +fn default_provider_fingerprints_remain_byte_stable() { + let relate_role = test_role(ModelRole::Relate, "relation/model"); + let relate_client = OpenRouterClient::from_role(&relate_role).expect("client"); + let relater = OpenRouterRelater { + client: relate_client, + role: relate_role, + }; + assert_eq!( + relater.fingerprint(), + "openrouter:relation/model|relation-v2" + ); + + let propose_role = test_role(ModelRole::Propose, "propose/model"); + let propose_client = OpenRouterClient::from_role(&propose_role).expect("client"); + let proposer = OpenRouterProposer { + client: propose_client, + role: propose_role, + }; + assert_eq!( + proposer.fingerprint(), + "openrouter:propose/model|propose-v3" + ); +} + +// --- embeddings parsing --- + +#[test] +fn embeddings_response_parses_in_request_order() { + let value = json!({ + "data": [ + { "index": 1, "embedding": [0.1, 0.2] }, + { "index": 0, "embedding": [0.3, 0.4] } + ] + }); + let vectors = parse_embeddings_response(&value, 2).expect("parse"); + // Sorted by `index`: row 0 first. + assert_eq!(vectors, vec![vec![0.3, 0.4], vec![0.1, 0.2]]); +} + +#[test] +fn embeddings_response_without_index_keeps_array_order() { + let value = json!({ + "data": [ + { "embedding": [1.0] }, + { "embedding": [2.0] } + ] + }); + let vectors = parse_embeddings_response(&value, 2).expect("parse"); + assert_eq!(vectors, vec![vec![1.0], vec![2.0]]); +} + +#[test] +fn embeddings_response_wrong_count_is_error() { + let value = json!({ "data": [ { "embedding": [1.0] } ] }); + let err = parse_embeddings_response(&value, 2).expect_err("count mismatch"); + assert!(matches!(err, BackendError::UnexpectedResponse { .. })); +} + +#[test] +fn embeddings_response_empty_vector_is_error() { + let value = json!({ "data": [ { "embedding": [] } ] }); + let err = parse_embeddings_response(&value, 1).expect_err("empty vector"); + assert!(matches!(err, BackendError::UnexpectedResponse { .. })); +} + +#[test] +fn embeddings_response_bad_shape_is_parse_error() { + let value = json!({ "data": [ { "embedding": "not-a-vector" } ] }); + let err = parse_embeddings_response(&value, 1).expect_err("parse error"); + assert!(matches!(err, BackendError::Parse { .. })); +} + +/// Helper that wraps a judge content string in a chat-completions envelope. +fn chat_envelope(content: &str) -> Value { + json!({ "choices": [ { "message": { "role": "assistant", "content": content } } ] }) +} + +fn chat_envelope_with_reason(content: &str, finish_reason: &str) -> Value { + json!({ + "choices": [{ + "finish_reason": finish_reason, + "message": { "role": "assistant", "content": content } + }] + }) +} + +#[test] +fn positive_length_finish_is_truncated_but_unknown_reason_is_preserved() { + let truncated = parse_relation_response(&chat_envelope_with_reason("", "length"), 4096) + .expect_err("positive truncation"); + assert!(matches!( + truncated, + BackendError::Truncated { + finish_reason, + max_tokens: 4096, + .. + } if finish_reason == "length" + )); + + let unknown = parse_relation_response(&chat_envelope_with_reason("", "weird_variant"), 4096) + .expect_err("unknown finish reason is malformed, not truncation"); + assert!(matches!(unknown, BackendError::UnexpectedResponse { .. })); + assert!(unknown.to_string().contains("weird_variant")); +} +// --- claim-relation parsing + label mapping --- + +#[test] +fn relation_request_carries_prompt_and_strict_format() { + let role = test_role(ModelRole::Relate, "nvidia/nemotron-3-ultra-550b-a55b"); + let body = build_relation_request(&role, "older X", "newer Y"); + assert_eq!(body["model"], "nvidia/nemotron-3-ultra-550b-a55b"); + assert_eq!(body["temperature"], json!(0.0)); + assert_eq!( + body["response_format"]["json_schema"]["name"], + "claim_relation" + ); + assert_eq!(body["max_tokens"], 4096); + let messages = body["messages"].as_array().expect("messages array"); + assert_eq!(messages.len(), 2); + let user = messages[1]["content"].as_str().expect("user content"); + assert!(user.contains("older X")); + assert!(user.contains("newer Y")); + // Order must be conveyed: older is labeled first, newer second. + assert!(user.find("older X") < user.find("newer Y")); +} + +#[test] +fn provider_capability_suppresses_unsupported_response_format() { + let mut role = test_role(ModelRole::Relate, "provider/model"); + role.profile.strict_json_schema_ok = false; + let body = build_relation_request(&role, "older", "newer"); + assert!(body.get("response_format").is_none()); +} + +#[test] +fn relation_label_strings_map_to_claim_relation() { + assert_eq!( + parse_relation_label("SUPERSEDES"), + Some(ClaimRelation::Supersedes) + ); + assert_eq!( + parse_relation_label(" conflict "), + Some(ClaimRelation::Conflict) + ); + assert_eq!( + parse_relation_label("duplicate"), + Some(ClaimRelation::Duplicate) + ); + assert_eq!( + parse_relation_label("unrelated"), + Some(ClaimRelation::Unrelated) + ); + assert_eq!(parse_relation_label("nonsense"), None); +} + +#[test] +fn relation_response_parses_clean_json() { + let value = chat_envelope("{\"relation\": \"supersedes\", \"score\": 0.88}"); + let verdict = parse_relation_response(&value, 4096).expect("parse"); + assert_eq!(verdict.relation, ClaimRelation::Supersedes); + assert!((verdict.score - 0.88).abs() < 1e-6); +} + +#[test] +fn relation_response_tolerates_fences_and_clamps_score() { + let value = chat_envelope("```json\n{\"relation\": \"conflict\", \"score\": 1.4}\n```"); + let verdict = parse_relation_response(&value, 4096).expect("parse"); + assert_eq!(verdict.relation, ClaimRelation::Conflict); + assert!((verdict.score - 1.0).abs() < 1e-6); +} + +#[test] +fn relation_response_unknown_label_is_error() { + let value = chat_envelope("{\"relation\": \"maybe\", \"score\": 0.5}"); + let err = parse_relation_response(&value, 4096).expect_err("unknown relation"); + assert!(matches!(err, BackendError::UnexpectedResponse { .. })); +} + +#[test] +fn relation_response_missing_score_is_parse_error() { + let value = chat_envelope("{\"relation\": \"unrelated\"}"); + let err = parse_relation_response(&value, 4096).expect_err("missing score"); + assert!(matches!(err, BackendError::Parse { .. })); +} + +// --- proposer parsing --- + +#[test] +fn propose_request_includes_heading_context_and_schema() { + let role = test_role(ModelRole::Propose, "anthropic/claude-opus-4.8"); + let body = build_propose_request( + &role, + "Deploys moved to Tuesday.", + &["Operations".to_owned(), "Deploys".to_owned()], + ); + assert_eq!(body["max_tokens"], 2048); + let user = body["messages"][1]["content"].as_str().expect("user"); + assert!(user.contains("Operations > Deploys"), "heading path joined"); + assert!(user.contains("Deploys moved to Tuesday.")); +} + +#[test] +fn propose_request_without_headings_has_no_section_line() { + let role = test_role(ModelRole::Propose, "m"); + let body = build_propose_request(&role, "Some span.", &[]); + let user = body["messages"][1]["content"].as_str().expect("user"); + assert!(!user.contains("Section:")); + assert!(user.starts_with("Span:")); +} + +#[test] +fn propose_response_parses_claims_and_rescales_confidence() { + let value = chat_envelope( + "{\"claims\":[{\"text\":\"Deploys moved to Tuesday.\",\"subject\":\"deploys\",\"predicate\":\"scheduled\",\"object\":\"Tuesday\",\"confidence\":90}]}", + ); + let claims = parse_propose_response(&value, 2048).expect("parse"); + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].text, "Deploys moved to Tuesday."); + assert_eq!(claims[0].subject, "deploys"); + assert_eq!(claims[0].confidence_ppm, 900_000); +} + +#[test] +fn propose_response_drops_blank_text_and_clamps_confidence() { + let value = chat_envelope( + "{\"claims\":[{\"text\":\" \",\"subject\":\"\",\"predicate\":\"\",\"object\":\"\",\"confidence\":50},{\"text\":\"Real claim.\",\"subject\":\"x\",\"predicate\":\"y\",\"object\":\"z\",\"confidence\":170}]}", + ); + let claims = parse_propose_response(&value, 2048).expect("parse"); + assert_eq!(claims.len(), 1, "blank-text claim dropped"); + assert_eq!(claims[0].text, "Real claim."); + assert_eq!( + claims[0].confidence_ppm, 1_000_000, + "over-100 confidence clamped" + ); +} + +#[test] +fn propose_response_empty_list_is_ok() { + let value = chat_envelope("{\"claims\":[]}"); + let claims = parse_propose_response(&value, 2048).expect("parse"); + assert!(claims.is_empty()); +} + +#[test] +fn propose_response_tolerates_fences() { + let value = chat_envelope( + "```json\n{\"claims\":[{\"text\":\"A.\",\"subject\":\"a\",\"predicate\":\"b\",\"object\":\"c\",\"confidence\":40}]}\n```", + ); + let claims = parse_propose_response(&value, 2048).expect("parse"); + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].confidence_ppm, 400_000); +} + +#[test] +fn propose_response_missing_claims_key_is_parse_error() { + let value = chat_envelope("{\"items\":[]}"); + let err = parse_propose_response(&value, 2048).expect_err("missing claims"); + assert!(matches!(err, BackendError::Parse { .. })); +} + +/// PROVES: the chat-completions response parsers are TOTAL functions on +/// arbitrary model output — they return `Ok` or a typed `Err`, never panic — +/// and when `Ok`, scores are clamped to `[0,1]` and confidence to `0..=1e6`. +/// The cargo-fuzz substitute for the judge/proposer parsers. +mod robustness { + use super::super::{parse_propose_response, parse_relation_response}; + use super::chat_envelope; + use proptest::prelude::*; + + proptest! { + #![proptest_config(ProptestConfig::with_cases(128))] + + #[test] + fn relation_parser_is_total(content in any::()) { + if let Ok(v) = parse_relation_response(&chat_envelope(&content), 4096) { + prop_assert!((0.0..=1.0).contains(&v.score)); + } + } + + #[test] + fn propose_parser_is_total(content in any::()) { + if let Ok(claims) = parse_propose_response(&chat_envelope(&content), 2048) { + for c in claims { + prop_assert!(c.confidence_ppm <= 1_000_000); + prop_assert!(!c.text.trim().is_empty()); + } + } + } + } +} diff --git a/src/semantics/pipeline.rs b/src/semantics/pipeline.rs index a641f06..f753339 100644 --- a/src/semantics/pipeline.rs +++ b/src/semantics/pipeline.rs @@ -5,11 +5,10 @@ //! and [`crate::conflicts::detect`]) with **meaning-based** logic driven by two //! injected backends: //! -//! * an [`Embedder`] — used for **cluster-first candidate generation**: claims -//! are clustered into connected components over the cosine-similarity graph -//! (see [`group_claims`]), and only *within-cluster* pairs that also pass a -//! coarse cosine prefilter ever reach the judge, so obviously-unrelated claims -//! never cost a judge call; +//! * an [`Embedder`] — used for deterministic, cursor-paged candidate +//! generation. Each pass examines a hard-bounded slice of the global pair +//! space and only pairs clearing the configured semantic floor reach the +//! judge; //! * a [`ClaimRelater`] — an LLM-as-judge that, for one candidate pair, makes the //! single richer call embeddings + 3-way NLI cannot: are the claims about the //! same subject, and does the newer one *update* the older (supersede) or merely @@ -23,15 +22,21 @@ //! logic can be proven deterministically with in-test stubs (no model, no //! network). -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::time::{Duration, Instant}; +use std::collections::BTreeMap; -use crate::events::ids::{conflict_id_from_pair, ClaimId, ConflictId, SourceId}; +#[cfg(test)] +use std::collections::BTreeSet; +#[cfg(test)] +use std::time::Duration; + +use crate::events::ids::{ClaimId, ConflictId, SourceId}; use crate::knowledge::{SourceSnapshotId, TemporalRelation}; -use crate::relate::settlement::{ - HeldDecision, PairFailureView, RelationFailureClass, UnresolvedPair, -}; -use crate::semantics::{cosine_similarity, ClaimRelater, ClaimRelation, Embedder, SemanticsError}; +#[cfg(test)] +use crate::relate::settlement::RelationFailureClass; +use crate::relate::settlement::{HeldDecision, UnresolvedPair}; +#[cfg(test)] +use crate::semantics::ClaimRelation; +use crate::semantics::{cosine_similarity, Embedder, SemanticsError}; /// Active lifecycle status for a claim view. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -61,11 +66,13 @@ pub struct LocalSequence(u64); impl LocalSequence { /// Construct a local sequence wrapper. + #[must_use] pub const fn new(value: u64) -> Self { Self(value) } /// Return the wrapped sequence. + #[must_use] pub const fn get(self) -> u64 { self.0 } @@ -263,16 +270,46 @@ fn sequence_rank(view: &ClaimView) -> u64 { /// boundary, not from these gates). #[derive(Debug, Clone, Copy)] pub struct RelateThresholds { - /// Link threshold for connected-component **clustering** (candidate - /// generation). Pairs split across clusters are never judged; see - /// [`relate_claims`]. Typically the `[semantics]` `cosine_threshold`. + /// Corpus similarity floor. Candidate paging uses the lower of this value + /// and [`Self::prefilter`] to preserve recall while bounding work. pub cluster: f32, - /// Coarse per-pair cosine **prefilter** applied within a cluster. Must sit - /// below the lowest same-subject similarity in the corpus (the relater does - /// the real separating), so it is intentionally lower than `cluster`. + /// Coarse per-pair cosine prefilter. The relater performs the final semantic + /// classification, so this value is a recall gate rather than an authority. pub prefilter: f32, } +/// Default maximum number of global pair slots examined by one relate +/// pass. The bound applies to settled and unsettled pairs alike. +pub const DEFAULT_CANDIDATE_PAIR_BUDGET: usize = 4_096; + +/// Opaque deterministic cursor into global pair enumeration. +/// +/// A cursor counts raw pair slots, before the cosine prefilter and duplicate +/// text gate. Consequently a page examines at most its configured budget even +/// when most pairs are filtered or already settled. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct CandidateCursor(u64); + +impl CandidateCursor { + /// Start at the first global pair. + #[must_use] + pub const fn start() -> Self { + Self(0) + } + + /// Restore a cursor returned by a prior partial outcome. + #[must_use] + pub const fn from_offset(offset: u64) -> Self { + Self(offset) + } + + /// Return the stable wire representation. + #[must_use] + pub const fn offset(self) -> u64 { + self.0 + } +} + /// Frozen source-order evidence used to orient semantic claim pairs. /// /// Claims without Git evidence retain journal observation order. Once either @@ -342,6 +379,38 @@ impl RelateTemporalPolicy { (Some(_), None) | (None, Some(_)) => Some(TemporalRelation::Unknown), } } + + /// Stable identity of every durable input that can change temporal pair orientation. + #[must_use] + pub(crate) fn campaign_identity_digest(&self) -> String { + let mut hasher = blake3::Hasher::new(); + hash_identity_field(&mut hasher, b"texo.relate.temporal-policy.v1"); + for (claim_id, snapshot_id) in &self.claim_snapshots { + hash_identity_field(&mut hasher, claim_id.as_bytes()); + hash_identity_field(&mut hasher, snapshot_id.as_bytes()); + } + for ((left, right), relation) in &self.snapshot_relations { + hash_identity_field(&mut hasher, left.as_bytes()); + hash_identity_field(&mut hasher, right.as_bytes()); + hasher.update(&[temporal_relation_tag(*relation)]); + } + hasher.finalize().to_hex().to_string() + } +} + +fn hash_identity_field(hasher: &mut blake3::Hasher, value: &[u8]) { + hasher.update(&(value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +const fn temporal_relation_tag(relation: TemporalRelation) -> u8 { + match relation { + TemporalRelation::Before => 1, + TemporalRelation::After => 2, + TemporalRelation::Same => 3, + TemporalRelation::Concurrent => 4, + TemporalRelation::Unknown => 5, + } } const fn invert_temporal_relation(relation: TemporalRelation) -> TemporalRelation { @@ -380,1917 +449,122 @@ pub struct PairJudgment { pub reused_authority: bool, } -/// Complete, partial, or fully held semantic pipeline result. +/// A completed semantic pass. Authority-bearing derived edges exist only here. #[derive(Debug, Default)] -pub struct RelateOutcome { - /// Decisions whose full evidence set is present. +pub struct CompleteRelateOutcome { + /// Decisions reduced from the complete authoritative verdict set. pub related: RelatedClaims, - /// Candidate pairs for which no verdict exists. + /// Decisions withheld by semantic ambiguity rather than pagination. + pub held: Vec, + /// Total authoritative candidate judgments reduced for this result. + pub candidate_pairs: usize, + /// Pair-slot examination ceiling used by the completing page. + pub candidate_pair_budget: usize, +} + +/// An incomplete semantic pass. It deliberately cannot carry publishable +/// supersession or conflict edges. +#[derive(Debug)] +pub struct PartialRelateOutcome { + /// Successful judgments acquired on this bounded page. Callers journal + /// these before resuming. + pub judgments: Vec, + /// Candidate pairs on this page for which no verdict exists. pub unresolved: Vec, - /// Decisions withheld by the tainted-claim holdback rule. + /// Every derived edge is represented as held until completion. pub held: Vec, + /// Number of raw global pair slots examined on this page. + pub candidate_pairs: usize, + /// Pair-slot examination ceiling used by this page. + pub candidate_pair_budget: usize, + /// Cursor for the next bounded page or earliest unresolved pair. + pub next_candidate_cursor: CandidateCursor, } -impl std::ops::Deref for RelateOutcome { - type Target = RelatedClaims; - - fn deref(&self) -> &Self::Target { - &self.related - } -} - -/// Relate claims by a single richer judgment per candidate pair. -/// -/// This is the primary relating entry point. A 3-way NLI label cannot distinguish -/// a value replacement from a genuine disagreement — measured against real models, -/// *both* are mutual contradiction, and embeddings alone cannot tell "Friday -/// deploy" from "Friday release". A [`ClaimRelater`] answers both questions -/// (shared subject? update or conflict?) at once. -/// -/// Candidate generation is **cluster-first** so the judge-call count scales with -/// cluster sizes, not with the corpus: -/// 1. Embed every claim once; cluster the claims into connected components of the -/// cosine-similarity graph at [`RelateThresholds::cluster`] (the same -/// clustering as [`group_claims`]). -/// 2. Within each cluster, consider only pairs whose cosine similarity is -/// `>= prefilter` — a *coarse* recall gate that should sit **below** the -/// lowest same-subject similarity in the corpus, never high enough to do the -/// separating itself (that is the relater's job). -/// 3. Order each surviving pair oldest→newest (by `receipt.sequence`, index as a -/// deterministic tiebreak) and ask the relater how the newer relates to the -/// older. Identical-normalized-text pairs are skipped as duplicates. -/// 4. [`ClaimRelation::Supersedes`] → the older is superseded; among all claims -/// that supersede it, the **newest** wins (one canonical edge per stale claim). -/// 5. [`ClaimRelation::Conflict`] → a candidate conflict, kept only if **neither** -/// side was superseded in step 4 (a superseded claim is no longer current). -/// -/// # Cross-cluster pairs are deliberately skipped +/// Typed complete-or-partial semantic pipeline result. /// -/// A pair whose claims land in different clusters is **never judged**, even when -/// its cosine similarity clears the prefilter — that skip is the point: it bounds -/// judge calls to `Σ (|cluster| choose 2)` (roughly `O(n · max_cluster)`) instead -/// of `O(n²)` over the whole corpus, which is what makes relate practical on -/// large corpora. Same-subject claims embed well above any sane cluster -/// threshold (and components link transitively), so a genuinely related pair -/// landing in two clusters means the cluster threshold is set above the corpus's -/// same-subject similarity floor — lower `[semantics]` `cosine_threshold` rather -/// than the prefilter. With `cluster <= prefilter` the judged pair set is -/// identical to the pre-clustering behavior (every pair passing the prefilter is -/// by definition intra-cluster). For any pair that *is* judged, the verdict and -/// event semantics are exactly those of the pre-clustering pipeline. -/// -/// Pure and deterministic for a given input order and backend behavior: -/// clustering, pair enumeration, and all output ordering depend only on slice -/// order and journal sequence, never on hash-map iteration order. -/// -/// # Errors -/// -/// Returns [`PipelineError::Semantics`] when the [`Embedder`] fails to embed the -/// claim texts or the [`ClaimRelater`] fails to judge a candidate pair. -pub fn relate_claims( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - relater: &dyn ClaimRelater, - thresholds: RelateThresholds, -) -> Result { - relate_claims_with_settled( - claims, - embedder, - relater, - thresholds, - &BTreeMap::new(), - Duration::MAX, - ) -} - -/// A candidate pair that survived clustering, the prefilter, and the -/// duplicate-text check, ordered oldest -> newest. -struct PendingPair { - old_idx: usize, - new_idx: usize, - older: ClaimId, - newer: ClaimId, - temporal_failure: Option, -} - -/// Verdict-acquisition result for one pending pair. Clone so a coalesced text -/// pair's single outcome fans to every logical pair that shares its texts. -#[derive(Clone)] -enum PairOutcome { - Judged(crate::semantics::RelationVerdict, bool), - Failed(PairFailureView), -} - -/// The failure view for a pair the wall budget cut off before it ran. -fn budget_exhausted() -> PairFailureView { - PairFailureView { - class: RelationFailureClass::BudgetExhausted, - endpoint: None, - status: None, - attempts: 0, - } -} - -/// Embed, cluster, and enumerate surviving candidate pairs in deterministic -/// order. Returns `None` when fewer than two claims exist. -fn prepare_pairs( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - temporal: &RelateTemporalPolicy, -) -> Result>, PipelineError> { - if claims.len() < 2 { - return Ok(None); - } - let texts: Vec<&str> = claims.iter().map(|(_, v)| embedding_text(v)).collect(); - let embeddings = embedder.embed_batch(&texts)?; - - // Cluster first: the judge only ever sees within-cluster pairs. - let clusters = similarity_components(&embeddings, thresholds.cluster); - let mut pending = Vec::new(); - for cluster in &clusters { - for (pos, &i) in cluster.iter().enumerate() { - for &j in &cluster[pos + 1..] { - // Cluster members are ascending, so i < j always holds here. - if cosine_similarity(&embeddings[i], &embeddings[j]) < thresholds.prefilter { - continue; - } - let (old_idx, new_idx, temporal_failure) = - order_pair(claims, i, j, settled, temporal); - if claims[old_idx].1.normalized_text == claims[new_idx].1.normalized_text { - continue; - } - pending.push(PendingPair { - old_idx, - new_idx, - older: claims[old_idx].0.clone(), - newer: claims[new_idx].0.clone(), - temporal_failure, - }); - } - } - } - Ok(Some(pending)) -} - -fn order_pair( - claims: &[(ClaimId, ClaimView)], - left: usize, - right: usize, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - temporal: &RelateTemporalPolicy, -) -> (usize, usize, Option) { - let left_id = &claims[left].0; - let right_id = &claims[right].0; - if settled.contains_key(&(left_id.clone(), right_id.clone())) { - return (left, right, None); - } - if settled.contains_key(&(right_id.clone(), left_id.clone())) { - return (right, left, None); - } - let sequence_order = || { - if (sequence_rank(&claims[left].1), left) <= (sequence_rank(&claims[right].1), right) { - (left, right) - } else { - (right, left) - } - }; - match temporal.compare_claims(left_id, right_id) { - None | Some(TemporalRelation::Same) => { - let (old, new) = sequence_order(); - (old, new, None) - } - Some(TemporalRelation::Before) => (left, right, None), - Some(TemporalRelation::After) => (right, left, None), - Some(TemporalRelation::Concurrent) => { - let (old, new) = sequence_order(); - (old, new, Some(RelationFailureClass::TemporalConcurrent)) - } - Some(TemporalRelation::Unknown) => { - let (old, new) = sequence_order(); - (old, new, Some(RelationFailureClass::TemporalUnknown)) - } - } +/// The enum shape makes publishing derived authority from a partial pass +/// unrepresentable: only [`CompleteRelateOutcome`] contains those edges. +#[derive(Debug)] +pub enum RelateOutcome { + /// All candidate pages and verdicts are complete. + Complete(CompleteRelateOutcome), + /// More bounded work or a retry is required. + Partial(PartialRelateOutcome), } -/// Acquire one pair's verdict on the calling thread: journal authority first, -/// then the wall budget, then a live judge call. -fn acquire_sequential( - claims: &[(ClaimId, ClaimView)], - pair: &PendingPair, - relater: &dyn ClaimRelater, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - started: Instant, - budget: Duration, -) -> PairOutcome { - let key = (pair.older.clone(), pair.newer.clone()); - if let Some(verdict) = settled.get(&key) { - // DECISION(campaign): any journaled judgment settles the logical pair. - // Explicit authority supersession is the future hook; model/config - // changes never re-judge here. - return PairOutcome::Judged(*verdict, true); - } - if let Some(class) = pair.temporal_failure { - return PairOutcome::Failed(PairFailureView { - class, - endpoint: None, - status: None, - attempts: 0, - }); - } - if started.elapsed() >= budget { - return PairOutcome::Failed(budget_exhausted()); - } - // Feed raw claim text: case and update wording carry the intent signal - // that normalized embedding text discards. - let old_view = &claims[pair.old_idx].1; - let new_view = &claims[pair.new_idx].1; - match relater.relate(&old_view.text, &new_view.text) { - Ok(verdict) => PairOutcome::Judged(verdict, false), - Err(error) => PairOutcome::Failed(classify_pair_failure(&error)), +impl Default for RelateOutcome { + fn default() -> Self { + Self::Complete(CompleteRelateOutcome { + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + ..CompleteRelateOutcome::default() + }) } } -/// Fold acquired outcomes into the deterministic decision reduction. Outcome -/// order equals pending order, so judgments/unresolved vectors — and therefore -/// journal append order — are byte-identical to the sequential path. -#[expect( - clippy::too_many_lines, - reason = "holdback and deterministic decision reduction form one state-machine fold" -)] -fn reduce_outcomes( - claims: &[(ClaimId, ClaimView)], - pending: Vec, - outcomes: Vec, - temporal: &RelateTemporalPolicy, -) -> RelateOutcome { - let mut winners: BTreeMap = BTreeMap::new(); - let mut ambiguous_winners = BTreeSet::new(); - let mut conflict_pairs: Vec<(usize, usize)> = Vec::new(); - let mut judgments = Vec::new(); - let mut unresolved = Vec::new(); - for (pair, outcome) in pending.into_iter().zip(outcomes) { - let old_view = &claims[pair.old_idx].1; - let new_view = &claims[pair.new_idx].1; - let (verdict, reused_authority) = match outcome { - PairOutcome::Failed(failure) => { - unresolved.push(unresolved_pair( - pair.older, pair.newer, old_view, new_view, failure, - )); - continue; - } - PairOutcome::Judged(verdict, reused) => (verdict, reused), - }; - judgments.push(PairJudgment { - older_claim: pair.older, - newer_claim: pair.newer, - verdict, - reused_authority, - }); - match verdict.relation { - ClaimRelation::Supersedes => { - let better = match winners.get(&pair.old_idx) { - None => true, - Some(&cur) => match compare_successors(claims, cur, pair.new_idx, temporal) { - SuccessorOrder::Candidate => true, - SuccessorOrder::Current => false, - SuccessorOrder::Ambiguous => { - ambiguous_winners.insert(pair.old_idx); - (sequence_rank(&claims[pair.new_idx].1), pair.new_idx) - > (sequence_rank(&claims[cur].1), cur) - } - }, - }; - if better { - winners.insert(pair.old_idx, pair.new_idx); - } - } - ClaimRelation::Conflict => { - conflict_pairs.push(( - pair.old_idx.min(pair.new_idx), - pair.old_idx.max(pair.new_idx), - )); - } - ClaimRelation::Duplicate | ClaimRelation::Unrelated => {} - } - } - - let mut tainted = unresolved - .iter() - .flat_map(|pair| [pair.old_claim.clone(), pair.new_claim.clone()]) - .collect::>(); - tainted.extend( - ambiguous_winners - .into_iter() - .map(|idx| claims[idx].0.clone()), - ); - let mut held = Vec::new(); - let mut superseded = HashSet::new(); - let mut supersessions = Vec::new(); - for (&old, &new) in &winners { - let (old_id, _) = &claims[old]; - let (new_id, new_view) = &claims[new]; - let reason = format!( - "superseded by {}:{}", - new_view.source_path, new_view.line_start - ); - if tainted.contains(old_id) { - held.push(HeldDecision::Supersession { - old_claim: old_id.clone(), - new_claim: new_id.clone(), - reason, - }); - } else { - superseded.insert(old_id.clone()); - supersessions.push((old_id.clone(), new_id.clone(), reason)); +impl RelateOutcome { + /// Borrow the complete state when all candidate pages settled. + #[must_use] + pub const fn complete(&self) -> Option<&CompleteRelateOutcome> { + match self { + Self::Complete(outcome) => Some(outcome), + Self::Partial(_) => None, } } - supersessions.sort_by(|a, b| { - a.0.as_str() - .cmp(b.0.as_str()) - .then_with(|| a.1.as_str().cmp(b.1.as_str())) - }); - let mut conflicts: Vec = Vec::new(); - let mut seen: HashSet = HashSet::new(); - for (i, j) in conflict_pairs { - let (a_id, a_view) = &claims[i]; - let (b_id, b_view) = &claims[j]; - let conflict_id = conflict_id_from_pair(a_id, b_id); - if !seen.insert(conflict_id.clone()) { - continue; - } - let entry = ConflictEntry { - conflict_id, - claim_a: a_id.clone(), - claim_b: b_id.clone(), - subject_hint: a_view.subject_hint.clone(), - reason: format!( - "contradictory current claims: \"{}\" vs \"{}\"", - a_view.text, b_view.text - ), - status: ConflictStatus::Open, - }; - if tainted.contains(a_id) || tainted.contains(b_id) { - held.push(HeldDecision::Conflict { - conflict_id: entry.conflict_id, - claim_a: entry.claim_a, - claim_b: entry.claim_b, - reason: entry.reason, - }); - } else if !superseded.contains(a_id) && !superseded.contains(b_id) { - conflicts.push(entry); + /// Borrow the partial state when more work is required. + #[must_use] + pub const fn partial(&self) -> Option<&PartialRelateOutcome> { + match self { + Self::Partial(outcome) => Some(outcome), + Self::Complete(_) => None, } } - conflicts.sort_by(|x, y| x.conflict_id.as_str().cmp(y.conflict_id.as_str())); - - RelateOutcome { - related: RelatedClaims { - supersessions, - conflicts, - judgments, - }, - unresolved, - held, - } -} -enum SuccessorOrder { - Current, - Candidate, - Ambiguous, -} - -fn compare_successors( - claims: &[(ClaimId, ClaimView)], - current: usize, - candidate: usize, - temporal: &RelateTemporalPolicy, -) -> SuccessorOrder { - match temporal.compare_claims(&claims[current].0, &claims[candidate].0) { - None | Some(TemporalRelation::Same) => { - if (sequence_rank(&claims[candidate].1), candidate) - > (sequence_rank(&claims[current].1), current) - { - SuccessorOrder::Candidate - } else { - SuccessorOrder::Current - } + /// Successful judgments acquired by this invocation. + #[must_use] + pub fn judgments(&self) -> &[PairJudgment] { + match self { + Self::Complete(outcome) => &outcome.related.judgments, + Self::Partial(outcome) => &outcome.judgments, } - Some(TemporalRelation::Before) => SuccessorOrder::Candidate, - Some(TemporalRelation::After) => SuccessorOrder::Current, - Some(TemporalRelation::Concurrent | TemporalRelation::Unknown) => SuccessorOrder::Ambiguous, } -} -/// Relate claims while reusing journal-authoritative verdicts and enforcing a -/// global wall-clock budget. Verdicts are acquired sequentially in pair order. -/// -/// # Errors -/// Embedding failures remain fatal because no candidate substrate exists. -pub fn relate_claims_with_settled( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - relater: &dyn ClaimRelater, - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - budget: Duration, -) -> Result { - relate_claims_with_settled_temporal( - claims, - embedder, - relater, - thresholds, - settled, - &RelateTemporalPolicy::default(), - budget, - ) -} - -/// Relate claims with journal authority and replayed source-order evidence. -/// -/// # Errors -/// Embedding failures remain fatal because no candidate substrate exists. -pub fn relate_claims_with_settled_temporal( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - relater: &dyn ClaimRelater, - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - temporal: &RelateTemporalPolicy, - budget: Duration, -) -> Result { - let started = Instant::now(); - let Some(pending) = prepare_pairs(claims, embedder, thresholds, settled, temporal)? else { - return Ok(RelateOutcome::default()); - }; - let outcomes = pending - .iter() - .map(|pair| acquire_sequential(claims, pair, relater, settled, started, budget)) - .collect::>(); - Ok(reduce_outcomes(claims, pending, outcomes, temporal)) -} - -/// Like [`relate_claims_with_settled`], but live judge calls fan out across -/// `concurrency` worker threads. Journal-authoritative verdicts resolve on the -/// calling thread; workers receive only genuinely unjudged pairs; results are -/// reassembled in pending order, so decision reduction and journal append -/// order are byte-identical to the sequential path for the same verdict set. -/// Under a wall budget, WHICH pairs get cut off is timing-dependent (as it -/// already is sequentially); a completed pass is fully deterministic. -/// -/// # Errors -/// Embedding failures remain fatal because no candidate substrate exists. -pub fn relate_claims_settled_parallel( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - relater: &(dyn ClaimRelater + Sync), - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - budget: Duration, - concurrency: usize, -) -> Result { - relate_claims_settled_parallel_temporal( - claims, - embedder, - relater, - thresholds, - settled, - ParallelRelateOptions { - temporal: &RelateTemporalPolicy::default(), - budget, - concurrency, - }, - ) -} - -/// Runtime controls for parallel semantic settlement. -#[derive(Debug, Clone, Copy)] -pub struct ParallelRelateOptions<'a> { - /// Replayed source-order policy. - pub temporal: &'a RelateTemporalPolicy, - /// Global wall-clock budget. - pub budget: Duration, - /// Maximum live judge workers. - pub concurrency: usize, -} - -/// Parallel relation acquisition with journal authority and replayed -/// source-order evidence. -/// -/// # Errors -/// Embedding failures remain fatal because no candidate substrate exists. -pub fn relate_claims_settled_parallel_temporal( - claims: &[(ClaimId, ClaimView)], - embedder: &dyn Embedder, - relater: &(dyn ClaimRelater + Sync), - thresholds: RelateThresholds, - settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, - options: ParallelRelateOptions<'_>, -) -> Result { - let ParallelRelateOptions { - temporal, - budget, - concurrency, - } = options; - let started = Instant::now(); - let Some(pending) = prepare_pairs(claims, embedder, thresholds, settled, temporal)? else { - return Ok(RelateOutcome::default()); - }; - if concurrency <= 1 { - let outcomes = pending - .iter() - .map(|pair| acquire_sequential(claims, pair, relater, settled, started, budget)) - .collect::>(); - return Ok(reduce_outcomes(claims, pending, outcomes, temporal)); - } - - let mut outcomes: Vec> = Vec::with_capacity(pending.len()); - outcomes.resize_with(pending.len(), || None); - - // Coalesce unsettled jobs by their exact relate() arguments. The disk cache - // keys on (fingerprint, older_text, newer_text), so two logical pairs with - // identical texts MUST make one model call and share the verdict: otherwise - // a cold parallel run makes duplicate paid calls, can receive different - // verdicts for identical prompts, and races writes to one cache tmp file. - // The representative is the lowest pending index in each group, so the - // choice is deterministic. Settled pairs resolve here and never dispatch. - let mut groups: BTreeMap<(&str, &str), Vec> = BTreeMap::new(); - for (idx, pair) in pending.iter().enumerate() { - if let Some(verdict) = settled.get(&(pair.older.clone(), pair.newer.clone())) { - outcomes[idx] = Some(PairOutcome::Judged(*verdict, true)); - continue; - } - if let Some(class) = pair.temporal_failure { - outcomes[idx] = Some(PairOutcome::Failed(PairFailureView { - class, - endpoint: None, - status: None, - attempts: 0, - })); - continue; + /// Unresolved pairs, present only for a partial result. + #[must_use] + pub fn unresolved(&self) -> &[UnresolvedPair] { + match self { + Self::Complete(_) => &[], + Self::Partial(outcome) => &outcome.unresolved, } - let texts = ( - claims[pair.old_idx].1.text.as_str(), - claims[pair.new_idx].1.text.as_str(), - ); - groups.entry(texts).or_default().push(idx); } - let mut representatives: Vec = groups.values().map(|members| members[0]).collect(); - // `BTreeMap` orders groups by text, but budget priority is part of the - // deterministic pair-enumeration contract. Restore pending-pair order - // after coalescing so a tight wall budget considers the same earliest - // logical pairs as the sequential path. - representatives.sort_unstable(); - - let mut rep_outcomes: BTreeMap = std::thread::scope(|scope| { - let (job_tx, job_rx) = flume::bounded::(concurrency); - let (result_tx, result_rx) = flume::unbounded::<(usize, PairOutcome)>(); - for _ in 0..concurrency { - let job_rx = job_rx.clone(); - let result_tx = result_tx.clone(); - let pending = &pending; - scope.spawn(move || { - while let Ok(idx) = job_rx.recv() { - let pair = &pending[idx]; - let outcome = if started.elapsed() >= budget { - PairOutcome::Failed(budget_exhausted()) - } else { - let old_view = &claims[pair.old_idx].1; - let new_view = &claims[pair.new_idx].1; - match relater.relate(&old_view.text, &new_view.text) { - Ok(verdict) => PairOutcome::Judged(verdict, false), - Err(error) => PairOutcome::Failed(classify_pair_failure(&error)), - } - }; - if result_tx.send((idx, outcome)).is_err() { - return; - } - } - }); - } - drop(job_rx); - drop(result_tx); - for idx in &representatives { - if job_tx.send(*idx).is_err() { - break; - } - } - drop(job_tx); - result_rx.iter().collect() - }); - // Fan each representative's single outcome to every member of its group. - for members in groups.values() { - let outcome = rep_outcomes - .remove(&members[0]) - .unwrap_or(PairOutcome::Failed(budget_exhausted())); - for &idx in members { - outcomes[idx] = Some(outcome.clone()); + /// Held decisions for either completion state. + #[must_use] + pub fn held(&self) -> &[HeldDecision] { + match self { + Self::Complete(outcome) => &outcome.held, + Self::Partial(outcome) => &outcome.held, } } - let outcomes = outcomes - .into_iter() - .map(|slot| slot.unwrap_or(PairOutcome::Failed(budget_exhausted()))) - .collect::>(); - Ok(reduce_outcomes(claims, pending, outcomes, temporal)) -} - -fn unresolved_pair( - old_claim: ClaimId, - new_claim: ClaimId, - old_view: &ClaimView, - new_view: &ClaimView, - failure: PairFailureView, -) -> UnresolvedPair { - UnresolvedPair { - old_claim, - new_claim, - old_ref: format!("{}:{}", old_view.source_path, old_view.line_start), - new_ref: format!("{}:{}", new_view.source_path, new_view.line_start), - failure, - } -} - -#[cfg(feature = "openrouter")] -pub(crate) fn classify_pair_failure(error: &SemanticsError) -> PairFailureView { - use crate::semantics::openrouter::BackendError; - use crate::surfaces::openai::ApiFailureKind; - - let SemanticsError::Backend { source } = error else { - return generic_pair_failure(); - }; - let Some(backend) = source.downcast_ref::() else { - return generic_pair_failure(); - }; - match backend { - BackendError::Http { source, .. } => PairFailureView { - class: match source.kind { - ApiFailureKind::HttpStatus => RelationFailureClass::HttpStatus, - ApiFailureKind::Transport => RelationFailureClass::Transport, - ApiFailureKind::DeadlineExceeded => RelationFailureClass::Deadline, - ApiFailureKind::BadResponseJson => RelationFailureClass::Parse, - }, - endpoint: Some(source.endpoint.to_string()), - status: source.status, - attempts: source.attempts, - }, - BackendError::Truncated { endpoint, .. } => PairFailureView { - class: RelationFailureClass::Truncated, - endpoint: Some((*endpoint).to_string()), - status: None, - attempts: 1, - }, - BackendError::Parse { endpoint, .. } - | BackendError::UnexpectedResponse { endpoint, .. } => PairFailureView { - class: RelationFailureClass::Parse, - endpoint: Some((*endpoint).to_string()), - status: None, - attempts: 1, - }, + /// True when another bounded page or retry is required. + #[must_use] + pub const fn is_partial(&self) -> bool { + matches!(self, Self::Partial(_)) } } -#[cfg(not(feature = "openrouter"))] -pub(crate) fn classify_pair_failure(_error: &SemanticsError) -> PairFailureView { - generic_pair_failure() -} - -fn generic_pair_failure() -> PairFailureView { - PairFailureView { - class: RelationFailureClass::Transport, - endpoint: None, - status: None, - attempts: 1, - } -} +mod candidate; +mod reduction; +mod runtime; #[cfg(test)] -mod tests { - use super::*; - - use crate::extract::normalize::normalize_line; - - /// Deterministic embedder driven by a fixed text -> vector table. - /// - /// Lookup is by the first table entry whose key is a case-insensitive - /// substring of the embedded text, so callers key on a distinctive phrase - /// from each claim. Texts with no matching key get a unique orthogonal basis - /// vector (never grouped with anything), making "unmapped" inputs inert - /// rather than accidentally similar. - struct FixedEmbedder { - table: Vec<(&'static str, Vec)>, - width: usize, - } - - impl FixedEmbedder { - fn new(table: Vec<(&'static str, Vec)>, width: usize) -> Self { - Self { table, width } - } - - /// One-hot vector for an unmapped text, derived from its byte sum so the - /// same text is stable but distinct texts rarely collide. - fn fallback(&self, text: &str) -> Vec { - let mut out = vec![0.0f32; self.width]; - let sum: usize = text.bytes().map(usize::from).sum(); - out[sum % self.width] = 1.0; - out - } - } - - impl Embedder for FixedEmbedder { - fn embed(&self, text: &str) -> Result, SemanticsError> { - let lower = text.to_ascii_lowercase(); - for (key, vector) in &self.table { - if lower.contains(&key.to_ascii_lowercase()) { - return Ok(vector.clone()); - } - } - Ok(self.fallback(text)) - } - } - - use crate::semantics::RelationVerdict; - - /// Deterministic relater driven by an `(older_sub, newer_sub) -> relation` - /// table. The first entry whose substrings match both the older premise and - /// the newer hypothesis wins; unmatched pairs are [`ClaimRelation::Unrelated`] - /// (the safe default — no edge, no conflict). Keyed on distinctive phrases. - struct ScriptedRelater { - table: Vec<(&'static str, &'static str, ClaimRelation)>, - } - - impl ScriptedRelater { - fn new(table: Vec<(&'static str, &'static str, ClaimRelation)>) -> Self { - Self { table } - } - } - - impl ClaimRelater for ScriptedRelater { - fn relate(&self, older: &str, newer: &str) -> Result { - let o = older.to_ascii_lowercase(); - let nw = newer.to_ascii_lowercase(); - for (older_sub, newer_sub, relation) in &self.table { - if o.contains(&older_sub.to_ascii_lowercase()) - && nw.contains(&newer_sub.to_ascii_lowercase()) - { - return Ok(RelationVerdict { - relation: *relation, - score: 1.0, - }); - } - } - Ok(RelationVerdict { - relation: ClaimRelation::Unrelated, - score: 1.0, - }) - } - fn fingerprint(&self) -> String { - "scripted".to_owned() - } - } - - /// Shorthand for [`RelateThresholds`]. Passing `cluster == prefilter` - /// reproduces the pre-clustering judged pair set exactly (every pair passing - /// the prefilter is intra-cluster by definition), which is what the original - /// single-threshold tests exercised. - fn th(cluster: f32, prefilter: f32) -> RelateThresholds { - RelateThresholds { cluster, prefilter } - } - - fn claim(id: &str, subject: &str, text: &str, sequence: u64) -> (ClaimId, ClaimView) { - let claim_id = ClaimId::try_from(id).expect("valid claim id"); - let view = ClaimView { - claim_id: claim_id.clone(), - workspace_id: "demo".to_string(), - source_id: SourceId::try_from("src_abc123def456").expect("valid source id"), - source_path: "x.md".to_string(), - line_start: u32::try_from(sequence).unwrap_or(u32::MAX), - line_end: u32::try_from(sequence).unwrap_or(u32::MAX), - text: text.to_string(), - normalized_text: normalize_line(text), - subject_hint: subject.to_string(), - predicate_hint: "unknown".to_string(), - object_hint: text.to_ascii_lowercase(), - confidence_ppm: 650_000, - extractor_kind: "test".to_string(), - status: ClaimStatus::Current, - receipt: receipt_view( - sequence.into(), - sequence, - "ClaimRecorded", - "workspace:demo", - id, - ), - supersedes: Vec::new(), - superseded_by: None, - }; - (claim_id, view) - } - - /// Build the embedder for the deploy-schedule scenario: the three deploy-day - /// claims plus the noise claim all sit in the same cluster (they are about the - /// deploy day), so grouping is purely about meaning while supersession is left - /// to NLI to decide. - fn deploy_embedder() -> FixedEmbedder { - FixedEmbedder::new( - vec![ - ("friday", vec![1.0, 0.0, 0.0]), - ("wednesday", vec![0.98, 0.10, 0.0]), - ("tuesday", vec![0.97, 0.12, 0.0]), - ("asked about the deploy day", vec![0.96, 0.14, 0.0]), - ], - 3, - ) - } - - #[test] - fn deploy_schedule_groups_three_days_and_noise_together() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Wednesday", 2), - claim("claim_cccccccccccc", "x", "Deploys moved to Tuesday", 3), - claim( - "claim_dddddddddddd", - "x", - "dave asked about the deploy day", - 2, - ), - ]; - let groups = group_claims(&claims, &deploy_embedder(), 0.9).expect("group"); - assert_eq!(groups.len(), 1, "all four cluster on deploy-day meaning"); - assert_eq!(groups[0].len(), 4); - } - - /// Embedder for the release scenario: the two release-schedule claims cluster - /// together, but "Bob owns release approval" is a DIFFERENT subject and must - /// land in its own group (the key dogfood trap — same word, different - /// meaning). - fn release_embedder() -> FixedEmbedder { - FixedEmbedder::new( - vec![ - ("releases happen on monday", vec![1.0, 0.0]), - ("go out on friday", vec![0.95, 0.05]), - ("bob owns release approval", vec![0.0, 1.0]), - ], - 2, - ) - } - - #[test] - fn release_schedule_splits_from_release_approval_by_meaning() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Releases happen on Monday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Releases go out on Friday", 2), - claim("claim_cccccccccccc", "x", "Bob owns release approval", 3), - ]; - let groups = group_claims(&claims, &release_embedder(), 0.9).expect("group"); - assert_eq!( - groups.len(), - 2, - "schedule and approval are different subjects" - ); - // The schedule pair groups together; approval is alone. - let sizes: Vec = { - let mut s: Vec = groups.iter().map(Vec::len).collect(); - s.sort_unstable(); - s - }; - assert_eq!(sizes, vec![1, 2]); - } - - #[test] - fn backend_error_propagates() { - struct FailingEmbedder; - impl Embedder for FailingEmbedder { - fn embed(&self, _text: &str) -> Result, SemanticsError> { - Err(SemanticsError::DimensionMismatch { - expected: 2, - actual: 1, - }) - } - } - let claims = vec![claim("claim_aaaaaaaaaaaa", "x", "anything", 1)]; - let err = group_claims(&claims, &FailingEmbedder, 0.9).expect_err("must propagate"); - assert!(matches!(err, PipelineError::Semantics(_))); - } - - #[test] - fn group_claims_empty_input_is_empty() { - let embedder = FixedEmbedder::new(Vec::new(), 2); - assert!(group_claims(&[], &embedder, 0.9).expect("group").is_empty()); - } - - #[test] - fn grouping_is_transitive_via_connected_components() { - // A links B, B links C, but A does not directly link C; connected - // components still place all three in one group. - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "alpha", 1), - claim("claim_bbbbbbbbbbbb", "x", "bravo", 2), - claim("claim_cccccccccccc", "x", "charlie", 3), - ]; - let embedder = FixedEmbedder::new( - vec![ - ("alpha", vec![1.0, 0.0]), - ("bravo", vec![0.95, 0.31]), - ("charlie", vec![0.80, 0.60]), - ], - 2, - ); - // alpha-bravo cosine ~0.95 (>=0.9), bravo-charlie ~0.95 (>=0.9), but - // alpha-charlie ~0.80 (<0.9): only connected components unite all three. - let groups = group_claims(&claims, &embedder, 0.9).expect("group"); - assert_eq!(groups.len(), 1, "transitive chain forms one component"); - assert_eq!(groups[0].len(), 3); - } - - // --- relate_claims (the LLM-relation-judge path) --- - - #[test] - fn relate_supersession_chain_picks_newest_winner_and_ignores_noise() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Wednesday", 2), - claim("claim_cccccccccccc", "x", "Deploys moved to Tuesday", 3), - claim( - "claim_dddddddddddd", - "x", - "dave asked about the deploy day", - 2, - ), - ]; - // The judge reports each newer deploy decision as superseding the older; - // the noise question is unrelated to every deploy claim. - let relater = ScriptedRelater::new(vec![ - ("friday", "wednesday", ClaimRelation::Supersedes), - ("friday", "tuesday", ClaimRelation::Supersedes), - ("wednesday", "tuesday", ClaimRelation::Supersedes), - ]); - let out = - relate_claims(&claims, &deploy_embedder(), &relater, th(0.9, 0.9)).expect("relate"); - - // Friday and Wednesday each superseded by Tuesday (the newest winner). - assert_eq!(out.supersessions.len(), 2); - let pairs: Vec<(&str, &str)> = out - .supersessions - .iter() - .map(|(o, n, _)| (o.as_str(), n.as_str())) - .collect(); - assert!(pairs.contains(&("claim_aaaaaaaaaaaa", "claim_cccccccccccc"))); - assert!(pairs.contains(&("claim_bbbbbbbbbbbb", "claim_cccccccccccc"))); - assert!( - !pairs - .iter() - .any(|(o, n)| *o == "claim_dddddddddddd" || *n == "claim_dddddddddddd"), - "noise never participates in supersession" - ); - assert!(out.conflicts.is_empty(), "no conflicts in a clean chain"); - } - - #[test] - fn relate_release_disagreement_is_conflict_not_supersession() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Releases happen on Monday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Releases go out on Friday", 2), - claim("claim_cccccccccccc", "x", "Bob owns release approval", 3), - ]; - // Monday vs Friday disagree with no update intent -> conflict. Approval is - // a different subject and never grouped with the schedule pair. - let relater = ScriptedRelater::new(vec![("monday", "friday", ClaimRelation::Conflict)]); - let out = - relate_claims(&claims, &release_embedder(), &relater, th(0.9, 0.9)).expect("relate"); - - assert!( - out.supersessions.is_empty(), - "a flat disagreement is not a supersession" - ); - assert_eq!(out.conflicts.len(), 1, "exactly one release conflict"); - let entry = &out.conflicts[0]; - let mut pair = [entry.claim_a.as_str(), entry.claim_b.as_str()]; - pair.sort_unstable(); - assert_eq!(pair, ["claim_aaaaaaaaaaaa", "claim_bbbbbbbbbbbb"]); - assert_eq!(entry.status, ConflictStatus::Open); - } - - #[test] - fn relate_superseded_claim_cannot_also_conflict() { - // A claim that is superseded must not surface as a live conflict, even if - // the judge also reports a contradicting peer. - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Tuesday", 3), - claim("claim_cccccccccccc", "x", "Deploys happen on Monday", 2), - ]; - let relater = ScriptedRelater::new(vec![ - ("friday", "tuesday", ClaimRelation::Supersedes), - ("monday", "tuesday", ClaimRelation::Supersedes), - ("friday", "monday", ClaimRelation::Conflict), - ]); - // All three deploy-day claims must cluster (deploy_embedder omits Monday). - let embedder = FixedEmbedder::new( - vec![ - ("friday", vec![1.0, 0.0, 0.0]), - ("tuesday", vec![0.97, 0.12, 0.0]), - ("monday", vec![0.96, 0.14, 0.0]), - ], - 3, - ); - let out = relate_claims(&claims, &embedder, &relater, th(0.9, 0.9)).expect("relate"); - // Friday and Monday are both superseded by Tuesday, so the Friday/Monday - // conflict is dropped. - assert_eq!(out.supersessions.len(), 2); - assert!( - out.conflicts.is_empty(), - "conflict involving a superseded claim is dropped" - ); - } - - #[test] - fn relate_duplicate_text_is_skipped() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "Deploys moved to Tuesday", 1), - claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Tuesday", 2), - ]; - let embedder = FixedEmbedder::new(vec![("tuesday", vec![1.0, 0.0])], 2); - // Even if the judge would fire, identical normalized text is never judged. - let relater = ScriptedRelater::new(vec![("tuesday", "tuesday", ClaimRelation::Supersedes)]); - let out = relate_claims(&claims, &embedder, &relater, th(0.9, 0.9)).expect("relate"); - assert!(out.supersessions.is_empty()); - assert!(out.conflicts.is_empty()); - } - - /// A relater that panics if ever called: proves a candidate-generation gate - /// (cluster split or prefilter) drops a pair before any judge call. - struct NeverRelater; - impl ClaimRelater for NeverRelater { - #[expect( - clippy::panic, - reason = "test guard: reaching the judge for a gated-out pair is the failure being detected" - )] - fn relate(&self, _older: &str, _newer: &str) -> Result { - panic!("relater must not be called for gated-out pairs"); - } - fn fingerprint(&self) -> String { - "never".to_owned() - } - } - - #[test] - fn relate_prefilter_skips_low_similarity_pairs_within_a_cluster() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "alpha subject", 1), - claim("claim_bbbbbbbbbbbb", "x", "beta subject", 2), - ]; - // Cosine ~0.30: above the cluster link threshold (0.2), so both claims - // share one cluster — but below the prefilter (0.5), so the pair must - // still be gated out before any judge call. - let embedder = FixedEmbedder::new( - vec![("alpha", vec![1.0, 0.0]), ("beta", vec![0.3, 0.954])], - 2, - ); - let out = relate_claims(&claims, &embedder, &NeverRelater, th(0.2, 0.5)).expect("relate"); - assert!(out.supersessions.is_empty()); - assert!(out.conflicts.is_empty()); - } - - #[test] - fn relate_skips_cross_cluster_pairs_without_judging() { - // Cosine ~0.87: the pair clears the coarse prefilter (0.6) — under the - // pre-clustering pipeline it WOULD have been judged — but it falls below - // the cluster link threshold (0.95), so the two claims land in different - // clusters and the pair is deliberately never judged. This is the - // documented cross-cluster skip that bounds judge calls. - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "alpha subject", 1), - claim("claim_bbbbbbbbbbbb", "x", "beta subject", 2), - ]; - let embedder = FixedEmbedder::new( - vec![("alpha", vec![1.0, 0.0]), ("beta", vec![0.87, 0.493])], - 2, - ); - let out = relate_claims(&claims, &embedder, &NeverRelater, th(0.95, 0.6)).expect("relate"); - assert!(out.supersessions.is_empty()); - assert!(out.conflicts.is_empty()); - } - - #[test] - fn relate_empty_and_singleton_inputs_are_inert() { - let embedder = FixedEmbedder::new(Vec::new(), 2); - let relater = ScriptedRelater::new(Vec::new()); - let empty = relate_claims(&[], &embedder, &relater, th(0.5, 0.5)).expect("relate empty"); - assert!(empty.supersessions.is_empty() && empty.conflicts.is_empty()); - - let one = vec![claim("claim_aaaaaaaaaaaa", "x", "Deploys on Tuesday", 1)]; - let single = relate_claims(&one, &embedder, &relater, th(0.5, 0.5)).expect("relate one"); - assert!(single.supersessions.is_empty() && single.conflicts.is_empty()); - } - - #[test] - fn relate_propagates_embedder_failure() { - struct FailingEmbedder; - impl Embedder for FailingEmbedder { - fn embed(&self, _text: &str) -> Result, SemanticsError> { - Err(SemanticsError::DimensionMismatch { - expected: 2, - actual: 1, - }) - } - } - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "one", 1), - claim("claim_bbbbbbbbbbbb", "x", "two", 2), - ]; - let relater = ScriptedRelater::new(Vec::new()); - let err = relate_claims(&claims, &FailingEmbedder, &relater, th(0.5, 0.5)) - .expect_err("propagate"); - assert!(matches!(err, PipelineError::Semantics(_))); - } - - // --- cluster-first candidate generation (the O(n²) fix) --- - - /// Deterministic relater that counts every judge call and always answers - /// [`ClaimRelation::Unrelated`], so candidate generation alone decides how - /// many calls happen. - struct CountingRelater { - calls: std::sync::atomic::AtomicUsize, - } - - impl CountingRelater { - fn new() -> Self { - Self { - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - fn count(&self) -> usize { - self.calls.load(std::sync::atomic::Ordering::SeqCst) - } - } - - impl ClaimRelater for CountingRelater { - fn relate(&self, _older: &str, _newer: &str) -> Result { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Ok(RelationVerdict { - relation: ClaimRelation::Unrelated, - score: 1.0, - }) - } - fn fingerprint(&self) -> String { - "counting".to_owned() - } - } - - /// Corpus of `k = 3` well-separated clusters of size `m = 3` (n = 9), as 2-D - /// unit vectors at angular offsets: within-cluster spread ≤ 2° (cosine - /// ≥ 0.999), adjacent clusters 30° apart (cosine ≈ 0.85–0.88 — above the 0.6 - /// prefilter, below the 0.98 cluster threshold), far clusters 60° apart - /// (cosine < 0.6). - fn three_cluster_corpus() -> (Vec<(ClaimId, ClaimView)>, FixedEmbedder) { - // (distinct keyword, degrees) — no keyword is a substring of another. - let spec: [(&str, f32); 9] = [ - ("aardvark", 0.0), - ("abacus", 1.0), - ("acorn", 2.0), - ("baboon", 30.0), - ("badger", 31.0), - ("bagel", 32.0), - ("cactus", 60.0), - ("camel", 61.0), - ("candle", 62.0), - ]; - let table: Vec<(&'static str, Vec)> = spec - .iter() - .map(|(key, deg)| { - let rad = deg.to_radians(); - (*key, vec![rad.cos(), rad.sin()]) - }) - .collect(); - let claims: Vec<(ClaimId, ClaimView)> = spec - .iter() - .enumerate() - .map(|(idx, (key, _))| { - let seq = u64::try_from(idx).expect("small index") + 1; - let id = format!("claim_{idx:012x}"); - claim(&id, "x", &format!("the {key} subject"), seq) - }) - .collect(); - (claims, FixedEmbedder::new(table, 2)) - } - - const CORPUS_THRESHOLDS: RelateThresholds = RelateThresholds { - cluster: 0.98, - prefilter: 0.6, - }; - - #[test] - fn judge_calls_bounded_by_within_cluster_pairs() { - let (claims, embedder) = three_cluster_corpus(); - let n = claims.len(); - - // Sanity: the corpus clusters as 3 components of 3 at the cluster - // threshold, so the within-cluster pair bound is Σ (3 choose 2) = 9. - let groups = group_claims(&claims, &embedder, CORPUS_THRESHOLDS.cluster).expect("group"); - let sizes: Vec = groups.iter().map(Vec::len).collect(); - assert_eq!(sizes, vec![3, 3, 3]); - let within_cluster_pairs: usize = sizes.iter().map(|m| m * (m - 1) / 2).sum(); - - // The pre-clustering pipeline judged every pair clearing the prefilter — - // count those pairs to show the bound is a strict improvement here. - let texts: Vec<&str> = claims.iter().map(|(_, v)| v.text.as_str()).collect(); - let embeddings = embedder.embed_batch(&texts).expect("embed"); - let mut prefilter_pairs = 0usize; - for i in 0..n { - for j in (i + 1)..n { - if cosine_similarity(&embeddings[i], &embeddings[j]) >= CORPUS_THRESHOLDS.prefilter - { - prefilter_pairs += 1; - } - } - } - - let relater = CountingRelater::new(); - let out = relate_claims(&claims, &embedder, &relater, CORPUS_THRESHOLDS).expect("relate"); - assert!(out.supersessions.is_empty() && out.conflicts.is_empty()); - - assert!( - relater.count() <= within_cluster_pairs, - "judge calls ({}) must be bounded by Σ (m_i choose 2) = {within_cluster_pairs}", - relater.count() - ); - assert_eq!( - relater.count(), - within_cluster_pairs, - "every distinct-text within-cluster pair passes the prefilter here" - ); - assert!( - relater.count() < prefilter_pairs, - "clustering must judge strictly fewer pairs than the prefilter alone \ - ({} vs {prefilter_pairs})", - relater.count() - ); - assert!( - prefilter_pairs < n * (n - 1) / 2, - "sanity: some pairs fall below the prefilter too" - ); - } - - #[test] - fn clustering_is_deterministic_across_runs() { - let (claims, embedder) = three_cluster_corpus(); - let first = group_claims(&claims, &embedder, CORPUS_THRESHOLDS.cluster).expect("group"); - for _ in 0..5 { - let again = group_claims(&claims, &embedder, CORPUS_THRESHOLDS.cluster).expect("group"); - assert_eq!(first, again, "same input must yield identical clusters"); - } - // Stable ordering: members ascend within a group; groups are ordered by - // their first (smallest) member index — never hash-map iteration order. - for group in &first { - assert!(group.windows(2).all(|w| w[0] < w[1])); - } - let firsts: Vec = first.iter().map(|g| g[0]).collect(); - assert!(firsts.windows(2).all(|w| w[0] < w[1])); - } - - #[test] - fn relate_is_deterministic_across_runs() { - let (claims, embedder) = three_cluster_corpus(); - // Script a supersession and a conflict inside two different clusters so - // both output vectors are non-empty. - let relater = ScriptedRelater::new(vec![ - ("aardvark", "acorn", ClaimRelation::Supersedes), - ("baboon", "bagel", ClaimRelation::Conflict), - ]); - let first = relate_claims(&claims, &embedder, &relater, CORPUS_THRESHOLDS).expect("relate"); - assert_eq!(first.supersessions.len(), 1); - assert_eq!(first.conflicts.len(), 1); - for _ in 0..5 { - let again = - relate_claims(&claims, &embedder, &relater, CORPUS_THRESHOLDS).expect("relate"); - assert_eq!( - first.supersessions, again.supersessions, - "supersessions must be identical across runs" - ); - let ids = |out: &RelatedClaims| -> Vec { - out.conflicts - .iter() - .map(|c| c.conflict_id.to_string()) - .collect() - }; - assert_eq!( - ids(&first), - ids(&again), - "conflict ids must be identical across runs" - ); - } - } - - #[test] - fn unresolved_pair_taints_and_holds_dependent_supersession() { - struct PartiallyFailingRelater; - impl ClaimRelater for PartiallyFailingRelater { - fn relate(&self, older: &str, newer: &str) -> Result { - if older.contains("alpha") && newer.contains("beta") { - return Err(SemanticsError::Backend { - source: Box::new(std::io::Error::other("judge failed")), - }); - } - Ok(RelationVerdict { - relation: if older.contains("alpha") && newer.contains("gamma") { - ClaimRelation::Supersedes - } else { - ClaimRelation::Unrelated - }, - score: 1.0, - }) - } - - fn fingerprint(&self) -> String { - "partial".to_string() - } - } - - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "alpha old", 1), - claim("claim_bbbbbbbbbbbb", "x", "beta uncertain", 2), - claim("claim_cccccccccccc", "x", "gamma winner", 3), - ]; - let embedder = FixedEmbedder::new( - vec![ - ("alpha", vec![1.0, 0.0]), - ("beta", vec![1.0, 0.0]), - ("gamma", vec![1.0, 0.0]), - ], - 2, - ); - let outcome = relate_claims(&claims, &embedder, &PartiallyFailingRelater, th(0.9, 0.9)) - .expect("partial outcome"); - assert_eq!(outcome.unresolved.len(), 1); - assert!(outcome.related.supersessions.is_empty()); - assert!(matches!( - outcome.held.as_slice(), - [HeldDecision::Supersession { old_claim, new_claim, .. }] - if old_claim.as_str() == "claim_aaaaaaaaaaaa" - && new_claim.as_str() == "claim_cccccccccccc" - )); - } - - #[test] - fn authoritative_pair_is_reused_without_judge_call() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "x", "alpha", 1), - claim("claim_bbbbbbbbbbbb", "x", "beta", 2), - claim("claim_cccccccccccc", "x", "gamma", 3), - ]; - let embedder = FixedEmbedder::new( - vec![ - ("alpha", vec![1.0, 0.0]), - ("beta", vec![1.0, 0.0]), - ("gamma", vec![1.0, 0.0]), - ], - 2, - ); - let relater = CountingRelater::new(); - let mut settled = BTreeMap::new(); - settled.insert( - (claims[0].0.clone(), claims[1].0.clone()), - RelationVerdict { - relation: ClaimRelation::Unrelated, - score: 1.0, - }, - ); - let outcome = relate_claims_with_settled( - &claims, - &embedder, - &relater, - th(0.9, 0.9), - &settled, - Duration::MAX, - ) - .expect("outcome"); - assert_eq!(relater.count(), 2); - assert_eq!( - outcome - .related - .judgments - .iter() - .filter(|judgment| judgment.reused_authority) - .count(), - 1 - ); - } - - /// Verdict depends only on the text pair, so a correct parallel run must - /// produce byte-identical output to the sequential run; call count exposes - /// duplicate model calls for identical-text pairs. - struct TextKeyedCountingRelater { - calls: std::sync::atomic::AtomicUsize, - } - - impl TextKeyedCountingRelater { - fn new() -> Self { - Self { - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - fn count(&self) -> usize { - self.calls.load(std::sync::atomic::Ordering::SeqCst) - } - } - - impl ClaimRelater for TextKeyedCountingRelater { - fn relate(&self, _older: &str, newer: &str) -> Result { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - // Deterministic in the text pair: "supersede" wording => Supersedes. - let relation = if newer.contains("moved") { - ClaimRelation::Supersedes - } else { - ClaimRelation::Unrelated - }; - Ok(RelationVerdict { - relation, - score: 1.0, - }) - } - fn fingerprint(&self) -> String { - "text-keyed-counting".to_owned() - } - } - - /// Two clusters, each three same-day claims: many intra-cluster pairs feed - /// the fan-out so worker interleaving is genuinely exercised. - fn parallel_fanout_corpus() -> (Vec<(ClaimId, ClaimView)>, FixedEmbedder) { - let claims = vec![ - claim( - "claim_0000000000a1", - "deploy", - "Deploys happen on Friday", - 1, - ), - claim( - "claim_0000000000a2", - "deploy", - "Deploys moved to Wednesday", - 2, - ), - claim( - "claim_0000000000a3", - "deploy", - "Deploys moved to Tuesday", - 3, - ), - claim( - "claim_0000000000b1", - "release", - "Releases happen on Monday", - 4, - ), - claim( - "claim_0000000000b2", - "release", - "Releases moved to Thursday", - 5, - ), - claim( - "claim_0000000000b3", - "release", - "Releases moved to Saturday", - 6, - ), - ]; - let embedder = FixedEmbedder::new( - vec![ - ("deploys happen on friday", vec![1.0, 0.0]), - ("deploys moved to wednesday", vec![0.999, 0.01]), - ("deploys moved to tuesday", vec![0.998, 0.02]), - ("releases happen on monday", vec![0.0, 1.0]), - ("releases moved to thursday", vec![0.01, 0.999]), - ("releases moved to saturday", vec![0.02, 0.998]), - ], - 2, - ); - (claims, embedder) - } - - #[test] - fn parallel_relate_equals_sequential_output() { - let (claims, embedder) = parallel_fanout_corpus(); - let seq = relate_claims_with_settled( - &claims, - &embedder, - &TextKeyedCountingRelater::new(), - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - ) - .expect("sequential"); - for concurrency in [2_usize, 4, 8] { - let par = relate_claims_settled_parallel( - &claims, - &embedder, - &TextKeyedCountingRelater::new(), - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - concurrency, - ) - .expect("parallel"); - assert_eq!( - format!("{seq:?}"), - format!("{par:?}"), - "parallel output diverged at concurrency {concurrency}" - ); - } - } - - #[test] - fn parallel_relate_coalesces_duplicate_text_pairs_to_one_call() { - // Four logical claim ids but only two distinct text pairs among the - // candidate pairs: identical texts must judge exactly once. - let claims = vec![ - claim("claim_00000000dup1", "x", "The service uses Postgres", 1), - claim("claim_00000000dup2", "x", "The service uses Postgres", 2), - claim("claim_00000000dup3", "x", "The service uses Redis", 3), - claim("claim_00000000dup4", "x", "The service uses Redis", 4), - ]; - let embedder = FixedEmbedder::new( - vec![ - ("the service uses postgres", vec![1.0, 0.0]), - ("the service uses redis", vec![0.99, 0.02]), - ], - 2, - ); - let relater = TextKeyedCountingRelater::new(); - let out = relate_claims_settled_parallel( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - 8, - ) - .expect("parallel"); - // All four claims cluster (postgres ~= redis at 0.9998). The two - // same-text pairs (postgres,postgres) and (redis,redis) are dropped by - // the normalized-text guard, leaving four distinct LOGICAL cross-pairs - // that all carry the identical (postgres, redis) text pair. Coalescing - // must judge that text pair exactly ONCE and fan the verdict to all - // four — without it a cold parallel run would make four paid calls. - assert_eq!( - relater.count(), - 1, - "identical-text logical pairs must coalesce to a single model call" - ); - assert_eq!( - out.related.judgments.len(), - 4, - "every surviving logical pair receives the shared verdict" - ); - assert!(out.unresolved.is_empty()); - - // Sanity: the sequential path with the same stub (no disk cache) makes - // one call per pair — proving the parallel coalescing, not the corpus, - // is what collapses the calls. - let seq_relater = TextKeyedCountingRelater::new(); - let _ = relate_claims_with_settled( - &claims, - &embedder, - &seq_relater, - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - ) - .expect("sequential"); - assert_eq!( - seq_relater.count(), - 4, - "uncached sequential path judges each of the four logical pairs" - ); - } - - #[test] - fn parallel_relate_preserves_settlement_and_holdback() { - let (claims, embedder) = parallel_fanout_corpus(); - // Pre-settle one pair from journal authority; it must not be re-judged. - let mut settled = BTreeMap::new(); - settled.insert( - (claims[0].0.clone(), claims[1].0.clone()), - RelationVerdict { - relation: ClaimRelation::Supersedes, - score: 1.0, - }, - ); - let relater = TextKeyedCountingRelater::new(); - let out = relate_claims_settled_parallel( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &settled, - Duration::MAX, - 4, - ) - .expect("parallel"); - assert_eq!( - out.related - .judgments - .iter() - .filter(|judgment| judgment.reused_authority) - .count(), - 1, - "the settled pair is reused, not re-judged" - ); - } - - /// A relater that forces calls to complete in reverse dispatch order. If - /// reduction depended on completion order the output would flip; proving - /// byte-identity under this adversarial schedule witnesses the fan-out's - /// deterministic reassembly contract. - struct ReverseOrderRelater { - gate: std::sync::Barrier, - ranks: BTreeMap<(String, String), usize>, - state: std::sync::Mutex, - ready: std::sync::Condvar, - } - - struct ReverseOrderState { - next_rank: Option, - completed: Vec, - } - - impl ReverseOrderRelater { - fn new(ordered_pairs: Vec<(String, String)>) -> Self { - let total = ordered_pairs.len(); - let ranks = ordered_pairs - .into_iter() - .enumerate() - .map(|(rank, pair)| (pair, rank)) - .collect(); - Self { - gate: std::sync::Barrier::new(total), - ranks, - state: std::sync::Mutex::new(ReverseOrderState { - next_rank: total.checked_sub(1), - completed: Vec::with_capacity(total), - }), - ready: std::sync::Condvar::new(), - } - } - - fn completed(&self) -> Vec { - self.state - .lock() - .expect("reverse-order state lock") - .completed - .clone() - } - } - - impl ClaimRelater for ReverseOrderRelater { - fn relate(&self, older: &str, newer: &str) -> Result { - let rank = *self - .ranks - .get(&(older.to_string(), newer.to_string())) - .expect("pair has a dispatch rank"); - // All representatives must be in flight before the highest rank is - // released. Each completion then unlocks exactly the preceding - // rank, forcing N-1..0 without sleeps or scheduler assumptions. - self.gate.wait(); - let mut state = self.state.lock().expect("reverse-order state lock"); - while state.next_rank != Some(rank) { - state = self.ready.wait(state).expect("reverse-order wait"); - } - state.completed.push(rank); - state.next_rank = rank.checked_sub(1); - self.ready.notify_all(); - drop(state); - let relation = if newer.contains("moved") { - ClaimRelation::Supersedes - } else { - ClaimRelation::Unrelated - }; - Ok(RelationVerdict { - relation, - score: 1.0, - }) - } - fn fingerprint(&self) -> String { - "reverse-order".to_owned() - } - } - - #[test] - fn parallel_reassembly_is_independent_of_completion_order() { - let (claims, embedder) = parallel_fanout_corpus(); - let seq = relate_claims_with_settled( - &claims, - &embedder, - &TextKeyedCountingRelater::new(), - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - ) - .expect("sequential"); - - let pending = prepare_pairs( - &claims, - &embedder, - th(0.9, 0.6), - &BTreeMap::new(), - &RelateTemporalPolicy::default(), - ) - .expect("prepare") - .expect("non-empty candidates"); - let mut seen = BTreeSet::new(); - let ordered_pairs = pending - .iter() - .filter_map(|pair| { - let texts = ( - claims[pair.old_idx].1.text.clone(), - claims[pair.new_idx].1.text.clone(), - ); - seen.insert(texts.clone()).then_some(texts) - }) - .collect::>(); - let representative_calls = ordered_pairs.len(); - assert!( - representative_calls >= 2, - "need real fan-out to test ordering" - ); - - let relater = ReverseOrderRelater::new(ordered_pairs); - let par = relate_claims_settled_parallel( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &BTreeMap::new(), - Duration::MAX, - representative_calls, - ) - .expect("parallel"); - assert_eq!( - format!("{seq:?}"), - format!("{par:?}"), - "reassembly must not depend on worker completion order" - ); - assert_eq!( - relater.completed(), - (0..representative_calls).rev().collect::>(), - "test harness must actually force reverse completion order" - ); - } - - #[test] - fn git_ancestry_orients_pairs_instead_of_ingest_sequence() { - let claims = vec![ - claim( - "claim_aaaaaaaaaaaa", - "storage", - "old storage uses postgres", - 20, - ), - claim( - "claim_bbbbbbbbbbbb", - "storage", - "new storage uses batpak", - 10, - ), - ]; - let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); - let relater = ScriptedRelater::new(vec![( - "old storage", - "new storage", - ClaimRelation::Supersedes, - )]); - let left = SourceSnapshotId::derive("ancestor"); - let right = SourceSnapshotId::derive("descendant"); - let mut temporal = RelateTemporalPolicy::default(); - temporal.bind_claim(&claims[0].0, &left); - temporal.bind_claim(&claims[1].0, &right); - temporal.insert_relation(&left, &right, TemporalRelation::Before); - - let out = relate_claims_with_settled_temporal( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &BTreeMap::new(), - &temporal, - Duration::MAX, - ) - .expect("relate"); - - assert_eq!( - out.supersessions, - vec![( - claims[0].0.clone(), - claims[1].0.clone(), - "superseded by x.md:10".to_string(), - )] - ); - assert_eq!(out.related.judgments[0].older_claim, claims[0].0); - assert_eq!(out.related.judgments[0].newer_claim, claims[1].0); - } - - #[test] - fn incomparable_or_missing_source_order_never_calls_the_judge() { - for (relation, expected) in [ - ( - Some(TemporalRelation::Concurrent), - RelationFailureClass::TemporalConcurrent, - ), - (None, RelationFailureClass::TemporalUnknown), - ] { - let claims = vec![ - claim( - "claim_aaaaaaaaaaaa", - "storage", - "old storage uses postgres", - 1, - ), - claim( - "claim_bbbbbbbbbbbb", - "storage", - "new storage uses batpak", - 2, - ), - ]; - let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); - let relater = CountingRelater::new(); - let left = SourceSnapshotId::derive("left"); - let right = SourceSnapshotId::derive("right"); - let mut temporal = RelateTemporalPolicy::default(); - temporal.bind_claim(&claims[0].0, &left); - temporal.bind_claim(&claims[1].0, &right); - if let Some(relation) = relation { - temporal.insert_relation(&left, &right, relation); - } - - let out = relate_claims_settled_parallel_temporal( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &BTreeMap::new(), - ParallelRelateOptions { - temporal: &temporal, - budget: Duration::MAX, - concurrency: 4, - }, - ) - .expect("relate"); - - assert_eq!(relater.count(), 0); - assert!(out.related.judgments.is_empty()); - assert_eq!(out.unresolved.len(), 1); - assert_eq!(out.unresolved[0].failure.class, expected); - assert!(out.supersessions.is_empty() && out.conflicts.is_empty()); - } - } - - #[test] - fn journal_authority_keeps_its_original_pair_direction() { - let claims = vec![ - claim("claim_aaaaaaaaaaaa", "storage", "storage alpha", 1), - claim("claim_bbbbbbbbbbbb", "storage", "storage beta", 2), - ]; - let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); - let relater = CountingRelater::new(); - let left = SourceSnapshotId::derive("left"); - let right = SourceSnapshotId::derive("right"); - let mut temporal = RelateTemporalPolicy::default(); - temporal.bind_claim(&claims[0].0, &left); - temporal.bind_claim(&claims[1].0, &right); - temporal.insert_relation(&left, &right, TemporalRelation::After); - let mut settled = BTreeMap::new(); - settled.insert( - (claims[0].0.clone(), claims[1].0.clone()), - RelationVerdict { - relation: ClaimRelation::Unrelated, - score: 1.0, - }, - ); - - let out = relate_claims_settled_parallel_temporal( - &claims, - &embedder, - &relater, - th(0.9, 0.6), - &settled, - ParallelRelateOptions { - temporal: &temporal, - budget: Duration::MAX, - concurrency: 4, - }, - ) - .expect("relate"); - - assert_eq!(relater.count(), 0); - assert_eq!(out.related.judgments[0].older_claim, claims[0].0); - assert_eq!(out.related.judgments[0].newer_claim, claims[1].0); - assert!(out.related.judgments[0].reused_authority); - } -} +use candidate::prepare_pairs; +pub(crate) use runtime::classify_pair_failure; +pub use runtime::{ + relate_claims, relate_claims_settled_parallel, relate_claims_settled_parallel_temporal, + relate_claims_with_settled, relate_claims_with_settled_temporal, ParallelRelateOptions, +}; +#[cfg(test)] +mod tests; diff --git a/src/semantics/pipeline/candidate.rs b/src/semantics/pipeline/candidate.rs new file mode 100644 index 0000000..9389c78 --- /dev/null +++ b/src/semantics/pipeline/candidate.rs @@ -0,0 +1,238 @@ +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use crate::events::ids::ClaimId; +use crate::knowledge::TemporalRelation; +use crate::relate::settlement::{PairFailureView, RelationFailureClass}; +use crate::semantics::{cosine_similarity, ClaimRelater, Embedder}; + +use super::runtime::classify_pair_failure; +use super::{ + embedding_text, sequence_rank, CandidateCursor, ClaimView, PipelineError, RelateTemporalPolicy, + RelateThresholds, +}; + +/// A candidate pair that survived the bounded page, cosine floor, and +/// duplicate-text check, ordered oldest -> newest. +#[derive(Clone)] +pub(super) struct PendingPair { + pub(super) old_idx: usize, + pub(super) new_idx: usize, + pub(super) older: ClaimId, + pub(super) newer: ClaimId, + pub(super) temporal_failure: Option, + pub(super) cursor: CandidateCursor, +} + +pub(super) struct PreparedPairs { + pub(super) pending: Vec, + pub(super) examined_pairs: usize, + pub(super) next_cursor: Option, + pub(super) claim_clusters: Vec, +} + +/// Verdict-acquisition result for one pending pair. Clone so a coalesced text +/// pair's single outcome fans to every logical pair that shares its texts. +#[derive(Clone)] +pub(super) enum PairOutcome { + Judged(crate::semantics::RelationVerdict, bool), + Failed(PairFailureView), +} + +/// The failure view for a pair the wall budget cut off before it ran. +pub(super) fn budget_exhausted() -> PairFailureView { + PairFailureView { + class: RelationFailureClass::BudgetExhausted, + endpoint: None, + status: None, + attempts: 0, + } +} + +/// Embed claims and enumerate one hard-bounded deterministic pair page. +/// Returns `None` when fewer than two claims exist. +pub(super) fn prepare_pairs( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + thresholds: RelateThresholds, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + temporal: &RelateTemporalPolicy, + candidate_pair_budget: usize, + candidate_cursor: CandidateCursor, +) -> Result, PipelineError> { + if claims.len() < 2 { + return Ok(None); + } + let texts: Vec<&str> = claims.iter().map(|(_, v)| embedding_text(v)).collect(); + let embeddings = embedder.embed_batch(&texts)?; + + // One deterministic global pair space makes both work and memory page + // bounded. The pair prefilter remains the semantic floor; paging replaces + // connected-component fan-out without weakening that explicit judge gate. + let clusters = vec![(0..claims.len()).collect::>()]; + let claim_clusters = vec![0_usize; claims.len()]; + let candidate_floor = thresholds.prefilter; + let total_pairs = clusters + .iter() + .map(|cluster| pair_count(cluster.len())) + .sum::(); + let start = candidate_cursor.offset().min(total_pairs); + let end = start + .saturating_add(u64::try_from(candidate_pair_budget).unwrap_or(u64::MAX)) + .min(total_pairs); + let mut pending = Vec::new(); + let mut position = seek_pair(&clusters, start); + let mut ordinal = start; + while ordinal < end { + let Some((cluster_idx, left_pos, right_pos)) = position else { + break; + }; + let cluster = &clusters[cluster_idx]; + let i = cluster[left_pos]; + let j = cluster[right_pos]; + if cosine_similarity(&embeddings[i], &embeddings[j]) >= candidate_floor { + let (old_idx, new_idx, temporal_failure) = order_pair(claims, i, j, settled, temporal); + if claims[old_idx].1.normalized_text != claims[new_idx].1.normalized_text { + pending.push(PendingPair { + old_idx, + new_idx, + older: claims[old_idx].0.clone(), + newer: claims[new_idx].0.clone(), + temporal_failure, + cursor: CandidateCursor::from_offset(ordinal), + }); + } + } + ordinal = ordinal.saturating_add(1); + position = next_pair(&clusters, cluster_idx, left_pos, right_pos); + } + Ok(Some(PreparedPairs { + pending, + examined_pairs: usize::try_from(end.saturating_sub(start)).unwrap_or(usize::MAX), + next_cursor: (end < total_pairs).then(|| CandidateCursor::from_offset(end)), + claim_clusters, + })) +} + +fn pair_count(len: usize) -> u64 { + let len = u64::try_from(len).unwrap_or(u64::MAX); + len.saturating_mul(len.saturating_sub(1)) / 2 +} + +fn seek_pair(clusters: &[Vec], mut offset: u64) -> Option<(usize, usize, usize)> { + for (cluster_idx, cluster) in clusters.iter().enumerate() { + let count = pair_count(cluster.len()); + if offset >= count { + offset -= count; + continue; + } + for left in 0..cluster.len().saturating_sub(1) { + let row = u64::try_from(cluster.len() - left - 1).unwrap_or(u64::MAX); + if offset < row { + let right = left + 1 + usize::try_from(offset).unwrap_or(usize::MAX); + return Some((cluster_idx, left, right)); + } + offset -= row; + } + } + None +} + +fn next_pair( + clusters: &[Vec], + cluster_idx: usize, + left: usize, + right: usize, +) -> Option<(usize, usize, usize)> { + let cluster = &clusters[cluster_idx]; + if right + 1 < cluster.len() { + return Some((cluster_idx, left, right + 1)); + } + if left + 2 < cluster.len() { + return Some((cluster_idx, left + 1, left + 2)); + } + clusters + .iter() + .enumerate() + .skip(cluster_idx + 1) + .find(|(_, candidate)| candidate.len() >= 2) + .map(|(next, _)| (next, 0, 1)) +} + +fn order_pair( + claims: &[(ClaimId, ClaimView)], + left: usize, + right: usize, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + temporal: &RelateTemporalPolicy, +) -> (usize, usize, Option) { + let left_id = &claims[left].0; + let right_id = &claims[right].0; + if settled.contains_key(&(left_id.clone(), right_id.clone())) { + return (left, right, None); + } + if settled.contains_key(&(right_id.clone(), left_id.clone())) { + return (right, left, None); + } + let sequence_order = || { + if (sequence_rank(&claims[left].1), left) <= (sequence_rank(&claims[right].1), right) { + (left, right) + } else { + (right, left) + } + }; + match temporal.compare_claims(left_id, right_id) { + None | Some(TemporalRelation::Same) => { + let (old, new) = sequence_order(); + (old, new, None) + } + Some(TemporalRelation::Before) => (left, right, None), + Some(TemporalRelation::After) => (right, left, None), + Some(TemporalRelation::Concurrent) => { + let (old, new) = sequence_order(); + (old, new, Some(RelationFailureClass::TemporalConcurrent)) + } + Some(TemporalRelation::Unknown) => { + let (old, new) = sequence_order(); + (old, new, Some(RelationFailureClass::TemporalUnknown)) + } + } +} + +/// Acquire one pair's verdict on the calling thread: journal authority first, +/// then the wall budget, then a live judge call. +pub(super) fn acquire_sequential( + claims: &[(ClaimId, ClaimView)], + pair: &PendingPair, + relater: &dyn ClaimRelater, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + started: Instant, + budget: Duration, +) -> PairOutcome { + let key = (pair.older.clone(), pair.newer.clone()); + if let Some(verdict) = settled.get(&key) { + // DECISION(campaign): any journaled judgment settles the logical pair. + // Explicit authority supersession is the future hook; model/config + // changes never re-judge here. + return PairOutcome::Judged(*verdict, true); + } + if let Some(class) = pair.temporal_failure { + return PairOutcome::Failed(PairFailureView { + class, + endpoint: None, + status: None, + attempts: 0, + }); + } + if started.elapsed() >= budget { + return PairOutcome::Failed(budget_exhausted()); + } + // Feed raw claim text: case and update wording carry the intent signal + // that normalized embedding text discards. + let old_view = &claims[pair.old_idx].1; + let new_view = &claims[pair.new_idx].1; + match relater.relate(&old_view.text, &new_view.text) { + Ok(verdict) => PairOutcome::Judged(verdict, false), + Err(error) => PairOutcome::Failed(classify_pair_failure(&error)), + } +} diff --git a/src/semantics/pipeline/reduction.rs b/src/semantics/pipeline/reduction.rs new file mode 100644 index 0000000..481b6c2 --- /dev/null +++ b/src/semantics/pipeline/reduction.rs @@ -0,0 +1,379 @@ +use super::candidate::{PairOutcome, PendingPair}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use crate::events::ids::{conflict_id_from_pair, ClaimId, ConflictId}; +use crate::knowledge::TemporalRelation; +use crate::relate::settlement::{HeldDecision, UnresolvedPair}; +use crate::semantics::ClaimRelation; + +use super::{ + sequence_rank, CandidateCursor, ClaimView, CompleteRelateOutcome, ConflictEntry, + ConflictStatus, PairJudgment, PartialRelateOutcome, RelateOutcome, RelateTemporalPolicy, + RelatedClaims, SupersessionEdge, +}; + +/// Fold acquired outcomes into the deterministic decision reduction. Outcome +/// order equals pending order, so judgments/unresolved vectors — and therefore +/// journal append order — are byte-identical to the sequential path. +fn reduce_outcomes( + claims: &[(ClaimId, ClaimView)], + pending: Vec, + outcomes: Vec, + temporal: &RelateTemporalPolicy, +) -> ReducedOutcome { + let reduction = collect_pair_reduction(claims, pending, outcomes, temporal); + let mut tainted = reduction + .unresolved + .iter() + .flat_map(|pair| [pair.old_claim.clone(), pair.new_claim.clone()]) + .collect::>(); + tainted.extend( + reduction + .ambiguous_winners + .into_iter() + .map(|idx| claims[idx].0.clone()), + ); + let (superseded, supersessions, mut held) = + reduce_supersessions(claims, &reduction.winners, &tainted); + let conflicts = reduce_conflicts( + claims, + reduction.conflict_pairs, + &tainted, + &superseded, + &mut held, + ); + ReducedOutcome { + related: RelatedClaims { + supersessions, + conflicts, + judgments: reduction.judgments, + }, + unresolved: reduction.unresolved, + held, + } +} + +struct ReducedOutcome { + related: RelatedClaims, + unresolved: Vec, + held: Vec, +} + +pub(super) struct PageCompletion<'a> { + pub(super) settled: &'a BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + pub(super) claim_clusters: &'a [usize], + pub(super) examined_pairs: usize, + pub(super) candidate_pair_budget: usize, + pub(super) next_cursor: Option, +} + +pub(super) fn finish_page( + claims: &[(ClaimId, ClaimView)], + pending: Vec, + outcomes: Vec, + temporal: &RelateTemporalPolicy, + page: &PageCompletion<'_>, +) -> RelateOutcome { + let retry_cursor = pending + .iter() + .zip(&outcomes) + .filter_map(|(pair, outcome)| { + matches!(outcome, PairOutcome::Failed(_)).then_some(pair.cursor) + }) + .min(); + let resume = retry_cursor.into_iter().chain(page.next_cursor).min(); + let page_reduction = reduce_outcomes(claims, pending, outcomes, temporal); + if let Some(next_candidate_cursor) = resume { + return RelateOutcome::Partial(partial_outcome( + page_reduction, + page.examined_pairs, + page.candidate_pair_budget, + next_candidate_cursor, + )); + } + + let fresh = page_reduction + .related + .judgments + .iter() + .filter(|judgment| !judgment.reused_authority) + .map(|judgment| (judgment.older_claim.clone(), judgment.newer_claim.clone())) + .collect::>(); + let mut authority = (*page.settled).clone(); + for judgment in &page_reduction.related.judgments { + authority.insert( + (judgment.older_claim.clone(), judgment.newer_claim.clone()), + judgment.verdict, + ); + } + let (all_pending, all_outcomes) = + authoritative_pairs(claims, &authority, &fresh, page.claim_clusters); + let complete = reduce_outcomes(claims, all_pending, all_outcomes, temporal); + RelateOutcome::Complete(CompleteRelateOutcome { + candidate_pairs: complete.related.judgments.len(), + candidate_pair_budget: page.candidate_pair_budget, + related: complete.related, + held: complete.held, + }) +} + +fn partial_outcome( + reduction: ReducedOutcome, + candidate_pairs: usize, + candidate_pair_budget: usize, + next_candidate_cursor: CandidateCursor, +) -> PartialRelateOutcome { + let ReducedOutcome { + related, + unresolved, + mut held, + } = reduction; + held.extend( + related + .supersessions + .into_iter() + .map( + |(old_claim, new_claim, reason)| HeldDecision::Supersession { + old_claim, + new_claim, + reason, + }, + ), + ); + held.extend( + related + .conflicts + .into_iter() + .map(|conflict| HeldDecision::Conflict { + conflict_id: conflict.conflict_id, + claim_a: conflict.claim_a, + claim_b: conflict.claim_b, + reason: conflict.reason, + }), + ); + PartialRelateOutcome { + judgments: related.judgments, + unresolved, + held, + candidate_pairs, + candidate_pair_budget, + next_candidate_cursor, + } +} + +fn authoritative_pairs( + claims: &[(ClaimId, ClaimView)], + authority: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + fresh: &BTreeSet<(ClaimId, ClaimId)>, + claim_clusters: &[usize], +) -> (Vec, Vec) { + let by_id = claims + .iter() + .enumerate() + .map(|(idx, (id, _))| (id.as_str(), idx)) + .collect::>(); + let mut rows = authority + .iter() + .filter_map(|((older, newer), verdict)| { + let old_idx = *by_id.get(older.as_str())?; + let new_idx = *by_id.get(newer.as_str())?; + Some(( + ( + claim_clusters[old_idx.min(new_idx)], + old_idx.min(new_idx), + old_idx.max(new_idx), + ), + PendingPair { + old_idx, + new_idx, + older: older.clone(), + newer: newer.clone(), + temporal_failure: None, + cursor: CandidateCursor::start(), + }, + PairOutcome::Judged(*verdict, !fresh.contains(&(older.clone(), newer.clone()))), + )) + }) + .collect::>(); + rows.sort_by_key(|row| row.0); + rows.into_iter() + .map(|(_, pair, outcome)| (pair, outcome)) + .unzip() +} + +struct PairReduction { + winners: BTreeMap, + ambiguous_winners: BTreeSet, + conflict_pairs: Vec<(usize, usize)>, + judgments: Vec, + unresolved: Vec, +} + +fn collect_pair_reduction( + claims: &[(ClaimId, ClaimView)], + pending: Vec, + outcomes: Vec, + temporal: &RelateTemporalPolicy, +) -> PairReduction { + let mut winners: BTreeMap = BTreeMap::new(); + let mut ambiguous_winners = BTreeSet::new(); + let mut conflict_pairs: Vec<(usize, usize)> = Vec::new(); + let mut judgments = Vec::new(); + let mut unresolved = Vec::new(); + for (pair, outcome) in pending.into_iter().zip(outcomes) { + let old_view = &claims[pair.old_idx].1; + let new_view = &claims[pair.new_idx].1; + let (verdict, reused_authority) = match outcome { + PairOutcome::Failed(failure) => { + unresolved.push(super::runtime::unresolved_pair( + pair.older, pair.newer, old_view, new_view, failure, + )); + continue; + } + PairOutcome::Judged(verdict, reused) => (verdict, reused), + }; + judgments.push(PairJudgment { + older_claim: pair.older, + newer_claim: pair.newer, + verdict, + reused_authority, + }); + match verdict.relation { + ClaimRelation::Supersedes => { + let better = match winners.get(&pair.old_idx) { + None => true, + Some(&cur) => match compare_successors(claims, cur, pair.new_idx, temporal) { + SuccessorOrder::Candidate => true, + SuccessorOrder::Current => false, + SuccessorOrder::Ambiguous => { + ambiguous_winners.insert(pair.old_idx); + (sequence_rank(&claims[pair.new_idx].1), pair.new_idx) + > (sequence_rank(&claims[cur].1), cur) + } + }, + }; + if better { + winners.insert(pair.old_idx, pair.new_idx); + } + } + ClaimRelation::Conflict => { + conflict_pairs.push(( + pair.old_idx.min(pair.new_idx), + pair.old_idx.max(pair.new_idx), + )); + } + ClaimRelation::Duplicate | ClaimRelation::Unrelated => {} + } + } + PairReduction { + winners, + ambiguous_winners, + conflict_pairs, + judgments, + unresolved, + } +} + +fn reduce_supersessions( + claims: &[(ClaimId, ClaimView)], + winners: &BTreeMap, + tainted: &BTreeSet, +) -> (HashSet, Vec, Vec) { + let mut held = Vec::new(); + let mut superseded = HashSet::new(); + let mut supersessions = Vec::new(); + for (&old, &new) in winners { + let (old_id, _) = &claims[old]; + let (new_id, new_view) = &claims[new]; + let reason = format!( + "superseded by {}:{}", + new_view.source_path, new_view.line_start + ); + if tainted.contains(old_id) { + held.push(HeldDecision::Supersession { + old_claim: old_id.clone(), + new_claim: new_id.clone(), + reason, + }); + } else { + superseded.insert(old_id.clone()); + supersessions.push((old_id.clone(), new_id.clone(), reason)); + } + } + supersessions.sort_by(|a, b| { + a.0.as_str() + .cmp(b.0.as_str()) + .then_with(|| a.1.as_str().cmp(b.1.as_str())) + }); + (superseded, supersessions, held) +} + +fn reduce_conflicts( + claims: &[(ClaimId, ClaimView)], + conflict_pairs: Vec<(usize, usize)>, + tainted: &BTreeSet, + superseded: &HashSet, + held: &mut Vec, +) -> Vec { + let mut conflicts: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for (i, j) in conflict_pairs { + let (a_id, a_view) = &claims[i]; + let (b_id, b_view) = &claims[j]; + let conflict_id = conflict_id_from_pair(a_id, b_id); + if !seen.insert(conflict_id.clone()) { + continue; + } + let entry = ConflictEntry { + conflict_id, + claim_a: a_id.clone(), + claim_b: b_id.clone(), + subject_hint: a_view.subject_hint.clone(), + reason: format!( + "contradictory current claims: \"{}\" vs \"{}\"", + a_view.text, b_view.text + ), + status: ConflictStatus::Open, + }; + if tainted.contains(a_id) || tainted.contains(b_id) { + held.push(HeldDecision::Conflict { + conflict_id: entry.conflict_id, + claim_a: entry.claim_a, + claim_b: entry.claim_b, + reason: entry.reason, + }); + } else if !superseded.contains(a_id) && !superseded.contains(b_id) { + conflicts.push(entry); + } + } + conflicts.sort_by(|x, y| x.conflict_id.as_str().cmp(y.conflict_id.as_str())); + conflicts +} + +enum SuccessorOrder { + Current, + Candidate, + Ambiguous, +} + +fn compare_successors( + claims: &[(ClaimId, ClaimView)], + current: usize, + candidate: usize, + temporal: &RelateTemporalPolicy, +) -> SuccessorOrder { + match temporal.compare_claims(&claims[current].0, &claims[candidate].0) { + None | Some(TemporalRelation::Same) => { + if (sequence_rank(&claims[candidate].1), candidate) + > (sequence_rank(&claims[current].1), current) + { + SuccessorOrder::Candidate + } else { + SuccessorOrder::Current + } + } + Some(TemporalRelation::Before) => SuccessorOrder::Candidate, + Some(TemporalRelation::After) => SuccessorOrder::Current, + Some(TemporalRelation::Concurrent | TemporalRelation::Unknown) => SuccessorOrder::Ambiguous, + } +} diff --git a/src/semantics/pipeline/runtime.rs b/src/semantics/pipeline/runtime.rs new file mode 100644 index 0000000..b725473 --- /dev/null +++ b/src/semantics/pipeline/runtime.rs @@ -0,0 +1,448 @@ +use super::candidate::{ + acquire_sequential, budget_exhausted, prepare_pairs, PairOutcome, PendingPair, PreparedPairs, +}; +use super::reduction::{finish_page, PageCompletion}; +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use crate::events::ids::ClaimId; +use crate::relate::settlement::{PairFailureView, RelationFailureClass, UnresolvedPair}; +use crate::semantics::{ClaimRelater, Embedder, SemanticsError}; + +use super::{ + CandidateCursor, ClaimView, PipelineError, RelateOutcome, RelateTemporalPolicy, + RelateThresholds, DEFAULT_CANDIDATE_PAIR_BUDGET, +}; + +/// Relate claims by a single richer judgment per candidate pair. +/// +/// This is the primary relating entry point. A 3-way NLI label cannot distinguish +/// a value replacement from a genuine disagreement — measured against real models, +/// *both* are mutual contradiction, and embeddings alone cannot tell "Friday +/// deploy" from "Friday release". A [`ClaimRelater`] answers both questions +/// (shared subject? update or conflict?) at once. +/// +/// Candidate generation is a deterministic page over the global pair space: +/// 1. Embed every claim once. +/// 2. Examine at most `candidate_pair_budget` raw pair slots from the supplied +/// [`CandidateCursor`]. +/// 3. Judge only pairs clearing the lower configured cosine floor. This is a +/// recall gate; the relater performs semantic classification. +/// 4. Order each surviving pair oldest→newest (by `receipt.sequence`, index as a +/// deterministic tiebreak) and ask the relater how the newer relates to the +/// older. Identical-normalized-text pairs are skipped as duplicates. +/// 5. [`ClaimRelation::Supersedes`] → the older is superseded; among all claims +/// that supersede it, the **newest** wins (one canonical edge per stale claim). +/// 6. [`ClaimRelation::Conflict`] → a candidate conflict, kept only if **neither** +/// side was superseded in step 4 (a superseded claim is no longer current). +/// +/// The cursor counts raw pair slots before similarity and duplicate-text gates, +/// so settled pairs and filtered pairs consume the same bounded work as live +/// pairs. Partial pages cannot expose derived authority and return the exact +/// cursor required to resume. +/// +/// Pure and deterministic for a given input order and backend behavior: +/// embedding, pair enumeration, and all output ordering depend only on slice +/// order and journal sequence, never on hash-map iteration order. +/// +/// # Errors +/// +/// Returns [`PipelineError::Semantics`] when the [`Embedder`] fails to embed the +/// claim texts or the [`ClaimRelater`] fails to judge a candidate pair. +pub fn relate_claims( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + relater: &dyn ClaimRelater, + thresholds: RelateThresholds, +) -> Result { + relate_claims_with_settled( + claims, + embedder, + relater, + thresholds, + &BTreeMap::new(), + Duration::MAX, + ) +} + +/// Relate claims while reusing journal-authoritative verdicts and enforcing a +/// global wall-clock budget. Verdicts are acquired sequentially in pair order. +/// +/// # Errors +/// Embedding failures remain fatal because no candidate substrate exists. +pub fn relate_claims_with_settled( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + relater: &dyn ClaimRelater, + thresholds: RelateThresholds, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + budget: Duration, +) -> Result { + relate_claims_with_settled_temporal( + claims, + embedder, + relater, + thresholds, + settled, + &RelateTemporalPolicy::default(), + budget, + ) +} + +/// Relate claims with journal authority and replayed source-order evidence. +/// +/// # Errors +/// Embedding failures remain fatal because no candidate substrate exists. +pub fn relate_claims_with_settled_temporal( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + relater: &dyn ClaimRelater, + thresholds: RelateThresholds, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + temporal: &RelateTemporalPolicy, + budget: Duration, +) -> Result { + let started = Instant::now(); + let Some(prepared) = prepare_pairs( + claims, + embedder, + thresholds, + settled, + temporal, + DEFAULT_CANDIDATE_PAIR_BUDGET, + CandidateCursor::start(), + )? + else { + return Ok(RelateOutcome::default()); + }; + let PreparedPairs { + pending, + examined_pairs, + next_cursor, + claim_clusters, + } = prepared; + let outcomes = pending + .iter() + .map(|pair| acquire_sequential(claims, pair, relater, settled, started, budget)) + .collect::>(); + Ok(finish_page( + claims, + pending, + outcomes, + temporal, + &PageCompletion { + settled, + claim_clusters: &claim_clusters, + examined_pairs, + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + next_cursor, + }, + )) +} + +/// Like [`relate_claims_with_settled`], but live judge calls fan out across +/// `concurrency` worker threads. Journal-authoritative verdicts resolve on the +/// calling thread; workers receive only genuinely unjudged pairs; results are +/// reassembled in pending order, so decision reduction and journal append +/// order are byte-identical to the sequential path for the same verdict set. +/// Under a wall budget, WHICH pairs get cut off is timing-dependent (as it +/// already is sequentially); a completed pass is fully deterministic. +/// +/// # Errors +/// Embedding failures remain fatal because no candidate substrate exists. +pub fn relate_claims_settled_parallel( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + relater: &(dyn ClaimRelater + Sync), + thresholds: RelateThresholds, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + budget: Duration, + concurrency: usize, +) -> Result { + relate_claims_settled_parallel_temporal( + claims, + embedder, + relater, + thresholds, + settled, + ParallelRelateOptions { + temporal: &RelateTemporalPolicy::default(), + budget, + concurrency, + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + candidate_cursor: CandidateCursor::start(), + }, + ) +} + +/// Runtime controls for parallel semantic settlement. +#[derive(Debug, Clone, Copy)] +pub struct ParallelRelateOptions<'a> { + /// Replayed source-order policy. + pub temporal: &'a RelateTemporalPolicy, + /// Global wall-clock budget. + pub budget: Duration, + /// Maximum live judge workers. + pub concurrency: usize, + /// Hard ceiling for raw global pair slots examined by this pass. + pub candidate_pair_budget: usize, + /// Cursor returned by the prior partial result. + pub candidate_cursor: CandidateCursor, +} + +/// Parallel relation acquisition with journal authority and replayed +/// source-order evidence. +/// +/// # Errors +/// Embedding failures remain fatal because no candidate substrate exists. +pub fn relate_claims_settled_parallel_temporal( + claims: &[(ClaimId, ClaimView)], + embedder: &dyn Embedder, + relater: &(dyn ClaimRelater + Sync), + thresholds: RelateThresholds, + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + options: ParallelRelateOptions<'_>, +) -> Result { + let ParallelRelateOptions { + temporal, + budget, + concurrency, + candidate_pair_budget, + candidate_cursor, + } = options; + let started = Instant::now(); + let Some(prepared) = prepare_pairs( + claims, + embedder, + thresholds, + settled, + temporal, + candidate_pair_budget, + candidate_cursor, + )? + else { + return Ok(RelateOutcome::default()); + }; + let PreparedPairs { + pending, + examined_pairs, + next_cursor, + claim_clusters, + } = prepared; + if concurrency <= 1 { + let outcomes = pending + .iter() + .map(|pair| acquire_sequential(claims, pair, relater, settled, started, budget)) + .collect::>(); + return Ok(finish_page( + claims, + pending, + outcomes, + temporal, + &PageCompletion { + settled, + claim_clusters: &claim_clusters, + examined_pairs, + candidate_pair_budget, + next_cursor, + }, + )); + } + + let outcomes = acquire_parallel( + claims, + &pending, + relater, + settled, + started, + budget, + concurrency, + ); + + let outcomes = outcomes + .into_iter() + .map(|slot| slot.unwrap_or(PairOutcome::Failed(budget_exhausted()))) + .collect::>(); + Ok(finish_page( + claims, + pending, + outcomes, + temporal, + &PageCompletion { + settled, + claim_clusters: &claim_clusters, + examined_pairs, + candidate_pair_budget, + next_cursor, + }, + )) +} + +fn acquire_parallel( + claims: &[(ClaimId, ClaimView)], + pending: &[PendingPair], + relater: &(dyn ClaimRelater + Sync), + settled: &BTreeMap<(ClaimId, ClaimId), crate::semantics::RelationVerdict>, + started: Instant, + budget: Duration, + concurrency: usize, +) -> Vec> { + let mut outcomes = vec![None; pending.len()]; + let mut groups: BTreeMap<(&str, &str), Vec> = BTreeMap::new(); + for (idx, pair) in pending.iter().enumerate() { + if let Some(verdict) = settled.get(&(pair.older.clone(), pair.newer.clone())) { + outcomes[idx] = Some(PairOutcome::Judged(*verdict, true)); + } else if let Some(class) = pair.temporal_failure { + outcomes[idx] = Some(PairOutcome::Failed(PairFailureView { + class, + endpoint: None, + status: None, + attempts: 0, + })); + } else { + let texts = ( + claims[pair.old_idx].1.text.as_str(), + claims[pair.new_idx].1.text.as_str(), + ); + groups.entry(texts).or_default().push(idx); + } + } + let mut representatives = groups + .values() + .map(|members| members[0]) + .collect::>(); + representatives.sort_unstable(); + let mut representative_outcomes = run_parallel_jobs( + claims, + pending, + relater, + started, + budget, + concurrency, + &representatives, + ); + for members in groups.values() { + let outcome = representative_outcomes + .remove(&members[0]) + .unwrap_or(PairOutcome::Failed(budget_exhausted())); + for &idx in members { + outcomes[idx] = Some(outcome.clone()); + } + } + outcomes +} + +fn run_parallel_jobs( + claims: &[(ClaimId, ClaimView)], + pending: &[PendingPair], + relater: &(dyn ClaimRelater + Sync), + started: Instant, + budget: Duration, + concurrency: usize, + representatives: &[usize], +) -> BTreeMap { + std::thread::scope(|scope| { + let (job_tx, job_rx) = flume::bounded::(concurrency); + let (result_tx, result_rx) = flume::unbounded::<(usize, PairOutcome)>(); + for _ in 0..concurrency { + let job_rx = job_rx.clone(); + let result_tx = result_tx.clone(); + scope.spawn(move || { + while let Ok(idx) = job_rx.recv() { + let pair = &pending[idx]; + let outcome = if started.elapsed() >= budget { + PairOutcome::Failed(budget_exhausted()) + } else { + match relater + .relate(&claims[pair.old_idx].1.text, &claims[pair.new_idx].1.text) + { + Ok(verdict) => PairOutcome::Judged(verdict, false), + Err(error) => PairOutcome::Failed(classify_pair_failure(&error)), + } + }; + if result_tx.send((idx, outcome)).is_err() { + return; + } + } + }); + } + drop(job_rx); + drop(result_tx); + for &idx in representatives { + if job_tx.send(idx).is_err() { + break; + } + } + drop(job_tx); + result_rx.iter().collect() + }) +} + +pub(super) fn unresolved_pair( + old_claim: ClaimId, + new_claim: ClaimId, + old_view: &ClaimView, + new_view: &ClaimView, + failure: PairFailureView, +) -> UnresolvedPair { + UnresolvedPair { + old_claim, + new_claim, + old_ref: format!("{}:{}", old_view.source_path, old_view.line_start), + new_ref: format!("{}:{}", new_view.source_path, new_view.line_start), + failure, + } +} + +#[cfg(feature = "openrouter")] +pub(crate) fn classify_pair_failure(error: &SemanticsError) -> PairFailureView { + use crate::semantics::openrouter::BackendError; + use crate::surfaces::openai::ApiFailureKind; + + let SemanticsError::Backend { source } = error else { + return generic_pair_failure(); + }; + let Some(backend) = source.downcast_ref::() else { + return generic_pair_failure(); + }; + match backend { + BackendError::Http { source, .. } => PairFailureView { + class: match source.kind { + ApiFailureKind::HttpStatus => RelationFailureClass::HttpStatus, + ApiFailureKind::Transport => RelationFailureClass::Transport, + ApiFailureKind::DeadlineExceeded => RelationFailureClass::Deadline, + ApiFailureKind::BadResponseJson => RelationFailureClass::Parse, + }, + endpoint: Some(source.endpoint.to_string()), + status: source.status, + attempts: source.attempts, + }, + BackendError::Truncated { endpoint, .. } => PairFailureView { + class: RelationFailureClass::Truncated, + endpoint: Some((*endpoint).to_string()), + status: None, + attempts: 1, + }, + BackendError::Parse { endpoint, .. } + | BackendError::UnexpectedResponse { endpoint, .. } => PairFailureView { + class: RelationFailureClass::Parse, + endpoint: Some((*endpoint).to_string()), + status: None, + attempts: 1, + }, + } +} + +#[cfg(not(feature = "openrouter"))] +pub(crate) fn classify_pair_failure(_error: &SemanticsError) -> PairFailureView { + generic_pair_failure() +} + +fn generic_pair_failure() -> PairFailureView { + PairFailureView { + class: RelationFailureClass::Transport, + endpoint: None, + status: None, + attempts: 1, + } +} diff --git a/src/semantics/pipeline/tests/basic.rs b/src/semantics/pipeline/tests/basic.rs new file mode 100644 index 0000000..38e9b3f --- /dev/null +++ b/src/semantics/pipeline/tests/basic.rs @@ -0,0 +1,319 @@ +use super::*; + +#[test] +fn deploy_schedule_groups_three_days_and_noise_together() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Wednesday", 2), + claim("claim_cccccccccccc", "x", "Deploys moved to Tuesday", 3), + claim( + "claim_dddddddddddd", + "x", + "dave asked about the deploy day", + 2, + ), + ]; + let groups = group_claims(&claims, &deploy_embedder(), 0.9).expect("group"); + assert_eq!(groups.len(), 1, "all four cluster on deploy-day meaning"); + assert_eq!(groups[0].len(), 4); +} + +/// Embedder for the release scenario: the two release-schedule claims cluster +/// together, but "Bob owns release approval" is a DIFFERENT subject and must +/// land in its own group (the key dogfood trap — same word, different +/// meaning). +fn release_embedder() -> FixedEmbedder { + FixedEmbedder::new( + vec![ + ("releases happen on monday", vec![1.0, 0.0]), + ("go out on friday", vec![0.95, 0.05]), + ("bob owns release approval", vec![0.0, 1.0]), + ], + 2, + ) +} + +#[test] +fn release_schedule_splits_from_release_approval_by_meaning() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Releases happen on Monday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Releases go out on Friday", 2), + claim("claim_cccccccccccc", "x", "Bob owns release approval", 3), + ]; + let groups = group_claims(&claims, &release_embedder(), 0.9).expect("group"); + assert_eq!( + groups.len(), + 2, + "schedule and approval are different subjects" + ); + // The schedule pair groups together; approval is alone. + let sizes: Vec = { + let mut s: Vec = groups.iter().map(Vec::len).collect(); + s.sort_unstable(); + s + }; + assert_eq!(sizes, vec![1, 2]); +} + +#[test] +fn backend_error_propagates() { + struct FailingEmbedder; + impl Embedder for FailingEmbedder { + fn embed(&self, _text: &str) -> Result, SemanticsError> { + Err(SemanticsError::DimensionMismatch { + expected: 2, + actual: 1, + }) + } + } + let claims = vec![claim("claim_aaaaaaaaaaaa", "x", "anything", 1)]; + let err = group_claims(&claims, &FailingEmbedder, 0.9).expect_err("must propagate"); + assert!(matches!(err, PipelineError::Semantics(_))); +} + +#[test] +fn group_claims_empty_input_is_empty() { + let embedder = FixedEmbedder::new(Vec::new(), 2); + assert!(group_claims(&[], &embedder, 0.9).expect("group").is_empty()); +} + +#[test] +fn grouping_is_transitive_via_connected_components() { + // A links B, B links C, but A does not directly link C; connected + // components still place all three in one group. + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "alpha", 1), + claim("claim_bbbbbbbbbbbb", "x", "bravo", 2), + claim("claim_cccccccccccc", "x", "charlie", 3), + ]; + let embedder = FixedEmbedder::new( + vec![ + ("alpha", vec![1.0, 0.0]), + ("bravo", vec![0.95, 0.31]), + ("charlie", vec![0.80, 0.60]), + ], + 2, + ); + // alpha-bravo cosine ~0.95 (>=0.9), bravo-charlie ~0.95 (>=0.9), but + // alpha-charlie ~0.80 (<0.9): only connected components unite all three. + let groups = group_claims(&claims, &embedder, 0.9).expect("group"); + assert_eq!(groups.len(), 1, "transitive chain forms one component"); + assert_eq!(groups[0].len(), 3); +} + +// --- relate_claims (the LLM-relation-judge path) --- + +#[test] +fn relate_supersession_chain_picks_newest_winner_and_ignores_noise() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Wednesday", 2), + claim("claim_cccccccccccc", "x", "Deploys moved to Tuesday", 3), + claim( + "claim_dddddddddddd", + "x", + "dave asked about the deploy day", + 2, + ), + ]; + // The judge reports each newer deploy decision as superseding the older; + // the noise question is unrelated to every deploy claim. + let relater = ScriptedRelater::new(vec![ + ("friday", "wednesday", ClaimRelation::Supersedes), + ("friday", "tuesday", ClaimRelation::Supersedes), + ("wednesday", "tuesday", ClaimRelation::Supersedes), + ]); + let out = relate_claims(&claims, &deploy_embedder(), &relater, th(0.9, 0.9)).expect("relate"); + let complete_related = related(&out).expect("complete"); + + // Friday and Wednesday each superseded by Tuesday (the newest winner). + assert_eq!(complete_related.supersessions.len(), 2); + let pairs: Vec<(&str, &str)> = complete_related + .supersessions + .iter() + .map(|(o, n, _)| (o.as_str(), n.as_str())) + .collect(); + assert!(pairs.contains(&("claim_aaaaaaaaaaaa", "claim_cccccccccccc"))); + assert!(pairs.contains(&("claim_bbbbbbbbbbbb", "claim_cccccccccccc"))); + assert!( + !pairs + .iter() + .any(|(o, n)| *o == "claim_dddddddddddd" || *n == "claim_dddddddddddd"), + "noise never participates in supersession" + ); + assert!( + complete_related.conflicts.is_empty(), + "no conflicts in a clean chain" + ); +} + +#[test] +fn relate_release_disagreement_is_conflict_not_supersession() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Releases happen on Monday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Releases go out on Friday", 2), + claim("claim_cccccccccccc", "x", "Bob owns release approval", 3), + ]; + // Monday vs Friday disagree with no update intent -> conflict. Approval is + // a different subject and never grouped with the schedule pair. + let relater = ScriptedRelater::new(vec![("monday", "friday", ClaimRelation::Conflict)]); + let out = relate_claims(&claims, &release_embedder(), &relater, th(0.9, 0.9)).expect("relate"); + + assert!( + related(&out).expect("complete").supersessions.is_empty(), + "a flat disagreement is not a supersession" + ); + assert_eq!( + related(&out).expect("complete").conflicts.len(), + 1, + "exactly one release conflict" + ); + let entry = &related(&out).expect("complete").conflicts[0]; + let mut pair = [entry.claim_a.as_str(), entry.claim_b.as_str()]; + pair.sort_unstable(); + assert_eq!(pair, ["claim_aaaaaaaaaaaa", "claim_bbbbbbbbbbbb"]); + assert_eq!(entry.status, ConflictStatus::Open); +} + +#[test] +fn relate_superseded_claim_cannot_also_conflict() { + // A claim that is superseded must not surface as a live conflict, even if + // the judge also reports a contradicting peer. + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Deploys happen on Friday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Tuesday", 3), + claim("claim_cccccccccccc", "x", "Deploys happen on Monday", 2), + ]; + let relater = ScriptedRelater::new(vec![ + ("friday", "tuesday", ClaimRelation::Supersedes), + ("monday", "tuesday", ClaimRelation::Supersedes), + ("friday", "monday", ClaimRelation::Conflict), + ]); + // All three deploy-day claims must cluster (deploy_embedder omits Monday). + let embedder = FixedEmbedder::new( + vec![ + ("friday", vec![1.0, 0.0, 0.0]), + ("tuesday", vec![0.97, 0.12, 0.0]), + ("monday", vec![0.96, 0.14, 0.0]), + ], + 3, + ); + let out = relate_claims(&claims, &embedder, &relater, th(0.9, 0.9)).expect("relate"); + // Friday and Monday are both superseded by Tuesday, so the Friday/Monday + // conflict is dropped. + assert_eq!(related(&out).expect("complete").supersessions.len(), 2); + assert!( + related(&out).expect("complete").conflicts.is_empty(), + "conflict involving a superseded claim is dropped" + ); +} + +#[test] +fn relate_duplicate_text_is_skipped() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "Deploys moved to Tuesday", 1), + claim("claim_bbbbbbbbbbbb", "x", "Deploys moved to Tuesday", 2), + ]; + let embedder = FixedEmbedder::new(vec![("tuesday", vec![1.0, 0.0])], 2); + // Even if the judge would fire, identical normalized text is never judged. + let relater = ScriptedRelater::new(vec![("tuesday", "tuesday", ClaimRelation::Supersedes)]); + let out = relate_claims(&claims, &embedder, &relater, th(0.9, 0.9)).expect("relate"); + assert!(related(&out).expect("complete").supersessions.is_empty()); + assert!(related(&out).expect("complete").conflicts.is_empty()); +} + +/// Call counter proving a candidate-generation gate drops a pair before the +/// judge boundary. +#[derive(Default)] +struct NeverRelater(std::cell::Cell); +impl ClaimRelater for NeverRelater { + fn relate(&self, _older: &str, _newer: &str) -> Result { + self.0.set(self.0.get().saturating_add(1)); + Ok(RelationVerdict { + relation: ClaimRelation::Unrelated, + score: 1.0, + }) + } + fn fingerprint(&self) -> String { + "never".to_owned() + } +} + +#[test] +fn relate_prefilter_skips_low_similarity_pairs_within_a_cluster() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "alpha subject", 1), + claim("claim_bbbbbbbbbbbb", "x", "beta subject", 2), + ]; + // Cosine ~0.30: above the cluster link threshold (0.2), so both claims + // share one cluster — but below the prefilter (0.5), so the pair must + // still be gated out before any judge call. + let embedder = FixedEmbedder::new( + vec![("alpha", vec![1.0, 0.0]), ("beta", vec![0.3, 0.954])], + 2, + ); + let relater = NeverRelater::default(); + let out = relate_claims(&claims, &embedder, &relater, th(0.2, 0.5)).expect("relate"); + assert_eq!(relater.0.get(), 0); + assert!(related(&out).expect("complete").supersessions.is_empty()); + assert!(related(&out).expect("complete").conflicts.is_empty()); +} + +#[test] +fn paged_relate_keeps_recall_below_the_old_cluster_cutoff() { + // Cosine ~0.87 clears the lower semantic floor even though it falls below + // the former connected-component threshold. Bounded paging controls cost + // without dropping the pair from semantic consideration. + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "alpha subject", 1), + claim("claim_bbbbbbbbbbbb", "x", "beta subject", 2), + ]; + let embedder = FixedEmbedder::new( + vec![("alpha", vec![1.0, 0.0]), ("beta", vec![0.87, 0.493])], + 2, + ); + let relater = NeverRelater::default(); + let out = relate_claims(&claims, &embedder, &relater, th(0.95, 0.6)).expect("relate"); + assert_eq!(relater.0.get(), 1); + assert!(related(&out).expect("complete").supersessions.is_empty()); + assert!(related(&out).expect("complete").conflicts.is_empty()); +} + +#[test] +fn relate_empty_and_singleton_inputs_are_inert() { + let embedder = FixedEmbedder::new(Vec::new(), 2); + let relater = ScriptedRelater::new(Vec::new()); + let empty = relate_claims(&[], &embedder, &relater, th(0.5, 0.5)).expect("relate empty"); + assert!( + related(&empty).expect("complete").supersessions.is_empty() + && related(&empty).expect("complete").conflicts.is_empty() + ); + + let one = vec![claim("claim_aaaaaaaaaaaa", "x", "Deploys on Tuesday", 1)]; + let single = relate_claims(&one, &embedder, &relater, th(0.5, 0.5)).expect("relate one"); + assert!( + related(&single).expect("complete").supersessions.is_empty() + && related(&single).expect("complete").conflicts.is_empty() + ); +} + +#[test] +fn relate_propagates_embedder_failure() { + struct FailingEmbedder; + impl Embedder for FailingEmbedder { + fn embed(&self, _text: &str) -> Result, SemanticsError> { + Err(SemanticsError::DimensionMismatch { + expected: 2, + actual: 1, + }) + } + } + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "one", 1), + claim("claim_bbbbbbbbbbbb", "x", "two", 2), + ]; + let relater = ScriptedRelater::new(Vec::new()); + let err = + relate_claims(&claims, &FailingEmbedder, &relater, th(0.5, 0.5)).expect_err("propagate"); + assert!(matches!(err, PipelineError::Semantics(_))); +} diff --git a/src/semantics/pipeline/tests/mod.rs b/src/semantics/pipeline/tests/mod.rs new file mode 100644 index 0000000..c44fca1 --- /dev/null +++ b/src/semantics/pipeline/tests/mod.rs @@ -0,0 +1,205 @@ +use super::*; + +use crate::extract::normalize::normalize_line; +use crate::semantics::ClaimRelater; + +/// Deterministic embedder driven by a fixed text -> vector table. +/// +/// Lookup is by the first table entry whose key is a case-insensitive +/// substring of the embedded text, so callers key on a distinctive phrase +/// from each claim. Texts with no matching key get a unique orthogonal basis +/// vector (never grouped with anything), making "unmapped" inputs inert +/// rather than accidentally similar. +struct FixedEmbedder { + table: Vec<(&'static str, Vec)>, + width: usize, +} + +impl FixedEmbedder { + fn new(table: Vec<(&'static str, Vec)>, width: usize) -> Self { + Self { table, width } + } + + /// One-hot vector for an unmapped text, derived from its byte sum so the + /// same text is stable but distinct texts rarely collide. + fn fallback(&self, text: &str) -> Vec { + let mut out = vec![0.0f32; self.width]; + let sum: usize = text.bytes().map(usize::from).sum(); + out[sum % self.width] = 1.0; + out + } +} + +impl Embedder for FixedEmbedder { + fn embed(&self, text: &str) -> Result, SemanticsError> { + let lower = text.to_ascii_lowercase(); + for (key, vector) in &self.table { + if lower.contains(&key.to_ascii_lowercase()) { + return Ok(vector.clone()); + } + } + Ok(self.fallback(text)) + } +} + +use crate::semantics::RelationVerdict; + +/// Deterministic relater driven by an `(older_sub, newer_sub) -> relation` +/// table. The first entry whose substrings match both the older premise and +/// the newer hypothesis wins; unmatched pairs are [`ClaimRelation::Unrelated`] +/// (the safe default — no edge, no conflict). Keyed on distinctive phrases. +struct ScriptedRelater { + table: Vec<(&'static str, &'static str, ClaimRelation)>, +} + +impl ScriptedRelater { + fn new(table: Vec<(&'static str, &'static str, ClaimRelation)>) -> Self { + Self { table } + } +} + +impl ClaimRelater for ScriptedRelater { + fn relate(&self, older: &str, newer: &str) -> Result { + let o = older.to_ascii_lowercase(); + let nw = newer.to_ascii_lowercase(); + for (older_sub, newer_sub, relation) in &self.table { + if o.contains(&older_sub.to_ascii_lowercase()) + && nw.contains(&newer_sub.to_ascii_lowercase()) + { + return Ok(RelationVerdict { + relation: *relation, + score: 1.0, + }); + } + } + Ok(RelationVerdict { + relation: ClaimRelation::Unrelated, + score: 1.0, + }) + } + fn fingerprint(&self) -> String { + "scripted".to_owned() + } +} + +/// Shorthand for [`RelateThresholds`]. Passing `cluster == prefilter` +/// reproduces the pre-clustering judged pair set exactly (every pair passing +/// the prefilter is intra-cluster by definition), which is what the original +/// single-threshold tests exercised. +fn th(cluster: f32, prefilter: f32) -> RelateThresholds { + RelateThresholds { cluster, prefilter } +} + +fn claim(id: &str, subject: &str, text: &str, sequence: u64) -> (ClaimId, ClaimView) { + let claim_id = ClaimId::try_from(id).expect("valid claim id"); + let view = ClaimView { + claim_id: claim_id.clone(), + workspace_id: "demo".to_string(), + source_id: SourceId::try_from("src_abc123def456").expect("valid source id"), + source_path: "x.md".to_string(), + line_start: u32::try_from(sequence).unwrap_or(u32::MAX), + line_end: u32::try_from(sequence).unwrap_or(u32::MAX), + text: text.to_string(), + normalized_text: normalize_line(text), + subject_hint: subject.to_string(), + predicate_hint: "unknown".to_string(), + object_hint: text.to_ascii_lowercase(), + confidence_ppm: 650_000, + extractor_kind: "test".to_string(), + status: ClaimStatus::Current, + receipt: receipt_view( + sequence.into(), + sequence, + "ClaimRecorded", + "workspace:demo", + id, + ), + supersedes: Vec::new(), + superseded_by: None, + }; + (claim_id, view) +} + +fn complete(outcome: &RelateOutcome) -> Option<&CompleteRelateOutcome> { + outcome.complete() +} + +fn partial(outcome: &RelateOutcome) -> Option<&PartialRelateOutcome> { + outcome.partial() +} + +fn related(outcome: &RelateOutcome) -> Option<&RelatedClaims> { + outcome.complete().map(|complete| &complete.related) +} + +/// Build the embedder for the deploy-schedule scenario: the three deploy-day +/// claims plus the noise claim all sit in the same cluster (they are about the +/// deploy day), so grouping is purely about meaning while supersession is left +/// to NLI to decide. +fn deploy_embedder() -> FixedEmbedder { + FixedEmbedder::new( + vec![ + ("friday", vec![1.0, 0.0, 0.0]), + ("wednesday", vec![0.98, 0.10, 0.0]), + ("tuesday", vec![0.97, 0.12, 0.0]), + ("asked about the deploy day", vec![0.96, 0.14, 0.0]), + ], + 3, + ) +} + +/// Deterministic relater that counts every judge call and always answers +/// [`ClaimRelation::Unrelated`]. +struct CountingRelater { + calls: std::sync::atomic::AtomicUsize, +} + +impl CountingRelater { + fn new() -> Self { + Self { + calls: std::sync::atomic::AtomicUsize::new(0), + } + } + fn count(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } +} + +impl ClaimRelater for CountingRelater { + fn relate(&self, _older: &str, _newer: &str) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(RelationVerdict { + relation: ClaimRelation::Unrelated, + score: 1.0, + }) + } + fn fingerprint(&self) -> String { + "counting".to_owned() + } +} + +struct FingerprintRelater { + inner: CountingRelater, + fingerprint: &'static str, +} + +impl ClaimRelater for FingerprintRelater { + fn relate(&self, older: &str, newer: &str) -> Result { + self.inner.relate(older, newer) + } + + fn fingerprint(&self) -> String { + self.fingerprint.to_string() + } +} + +// Corpus of `k = 3` well-separated clusters of size `m = 3` (n = 9), as 2-D +// unit vectors at angular offsets: within-cluster spread ≤ 2° (cosine +// ≥ 0.999), adjacent clusters 30° apart (cosine ≈ 0.85–0.88 — above the 0.6 +// prefilter, below the 0.98 cluster threshold), far clusters 60° apart +// (cosine < 0.6). + +mod basic; +mod settlement; +mod parallel; +mod temporal; diff --git a/src/semantics/pipeline/tests/parallel.rs b/src/semantics/pipeline/tests/parallel.rs new file mode 100644 index 0000000..0efab3a --- /dev/null +++ b/src/semantics/pipeline/tests/parallel.rs @@ -0,0 +1,364 @@ +use super::*; + +/// Verdict depends only on the text pair; call count exposes duplicate model +/// calls for identical-text pairs. +struct TextKeyedCountingRelater { + calls: std::sync::atomic::AtomicUsize, +} + +impl TextKeyedCountingRelater { + fn new() -> Self { + Self { + calls: std::sync::atomic::AtomicUsize::new(0), + } + } + fn count(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } +} + +impl ClaimRelater for TextKeyedCountingRelater { + fn relate(&self, _older: &str, newer: &str) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Deterministic in the text pair: "supersede" wording => Supersedes. + let relation = if newer.contains("moved") { + ClaimRelation::Supersedes + } else { + ClaimRelation::Unrelated + }; + Ok(RelationVerdict { + relation, + score: 1.0, + }) + } + fn fingerprint(&self) -> String { + "text-keyed-counting".to_owned() + } +} + +/// Two clusters, each three same-day claims: many intra-cluster pairs feed +/// the fan-out so worker interleaving is genuinely exercised. +fn parallel_fanout_corpus() -> (Vec<(ClaimId, ClaimView)>, FixedEmbedder) { + let claims = vec![ + claim( + "claim_0000000000a1", + "deploy", + "Deploys happen on Friday", + 1, + ), + claim( + "claim_0000000000a2", + "deploy", + "Deploys moved to Wednesday", + 2, + ), + claim( + "claim_0000000000a3", + "deploy", + "Deploys moved to Tuesday", + 3, + ), + claim( + "claim_0000000000b1", + "release", + "Releases happen on Monday", + 4, + ), + claim( + "claim_0000000000b2", + "release", + "Releases moved to Thursday", + 5, + ), + claim( + "claim_0000000000b3", + "release", + "Releases moved to Saturday", + 6, + ), + ]; + let embedder = FixedEmbedder::new( + vec![ + ("deploys happen on friday", vec![1.0, 0.0]), + ("deploys moved to wednesday", vec![0.999, 0.01]), + ("deploys moved to tuesday", vec![0.998, 0.02]), + ("releases happen on monday", vec![0.0, 1.0]), + ("releases moved to thursday", vec![0.01, 0.999]), + ("releases moved to saturday", vec![0.02, 0.998]), + ], + 2, + ); + (claims, embedder) +} + +#[test] +fn parallel_relate_equals_sequential_output() { + let (claims, embedder) = parallel_fanout_corpus(); + let seq = relate_claims_with_settled( + &claims, + &embedder, + &TextKeyedCountingRelater::new(), + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + ) + .expect("sequential"); + for concurrency in [2_usize, 4, 8] { + let par = relate_claims_settled_parallel( + &claims, + &embedder, + &TextKeyedCountingRelater::new(), + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + concurrency, + ) + .expect("parallel"); + assert_eq!( + format!("{seq:?}"), + format!("{par:?}"), + "parallel output diverged at concurrency {concurrency}" + ); + } +} + +#[test] +fn parallel_relate_coalesces_duplicate_text_pairs_to_one_call() { + // Four logical claim ids but only two distinct text pairs among the + // candidate pairs: identical texts must judge exactly once. + let claims = vec![ + claim("claim_00000000dup1", "x", "The service uses Postgres", 1), + claim("claim_00000000dup2", "x", "The service uses Postgres", 2), + claim("claim_00000000dup3", "x", "The service uses Redis", 3), + claim("claim_00000000dup4", "x", "The service uses Redis", 4), + ]; + let embedder = FixedEmbedder::new( + vec![ + ("the service uses postgres", vec![1.0, 0.0]), + ("the service uses redis", vec![0.99, 0.02]), + ], + 2, + ); + let relater = TextKeyedCountingRelater::new(); + let out = relate_claims_settled_parallel( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + 8, + ) + .expect("parallel"); + // All four claims cluster (postgres ~= redis at 0.9998). The two + // same-text pairs (postgres,postgres) and (redis,redis) are dropped by + // the normalized-text guard, leaving four distinct LOGICAL cross-pairs + // that all carry the identical (postgres, redis) text pair. Coalescing + // must judge that text pair exactly ONCE and fan the verdict to all + // four — without it a cold parallel run would make four paid calls. + assert_eq!( + relater.count(), + 1, + "identical-text logical pairs must coalesce to a single model call" + ); + assert_eq!( + out.judgments().len(), + 4, + "every surviving logical pair receives the shared verdict" + ); + assert!(out.unresolved().is_empty()); + + // Sanity: the sequential path with the same stub (no disk cache) makes + // one call per pair — proving the parallel coalescing, not the corpus, + // is what collapses the calls. + let seq_relater = TextKeyedCountingRelater::new(); + let _ = relate_claims_with_settled( + &claims, + &embedder, + &seq_relater, + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + ) + .expect("sequential"); + assert_eq!( + seq_relater.count(), + 4, + "uncached sequential path judges each of the four logical pairs" + ); +} + +#[test] +fn parallel_relate_preserves_settlement_and_holdback() { + let (claims, embedder) = parallel_fanout_corpus(); + // Pre-settle one pair from journal authority; it must not be re-judged. + let mut settled = BTreeMap::new(); + settled.insert( + (claims[0].0.clone(), claims[1].0.clone()), + RelationVerdict { + relation: ClaimRelation::Supersedes, + score: 1.0, + }, + ); + let relater = TextKeyedCountingRelater::new(); + let out = relate_claims_settled_parallel( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &settled, + Duration::MAX, + 4, + ) + .expect("parallel"); + assert_eq!( + out.judgments() + .iter() + .filter(|judgment| judgment.reused_authority) + .count(), + 1, + "the settled pair is reused, not re-judged" + ); +} + +/// A relater that forces calls to complete in reverse dispatch order. If +/// reduction depended on completion order the output would flip; proving +/// byte-identity under this adversarial schedule witnesses the fan-out's +/// deterministic reassembly contract. +struct ReverseOrderRelater { + gate: std::sync::Barrier, + ranks: BTreeMap<(String, String), usize>, + state: std::sync::Mutex, + ready: std::sync::Condvar, +} + +struct ReverseOrderState { + next_rank: Option, + completed: Vec, +} + +impl ReverseOrderRelater { + fn new(ordered_pairs: Vec<(String, String)>) -> Self { + let total = ordered_pairs.len(); + let ranks = ordered_pairs + .into_iter() + .enumerate() + .map(|(rank, pair)| (pair, rank)) + .collect(); + Self { + gate: std::sync::Barrier::new(total), + ranks, + state: std::sync::Mutex::new(ReverseOrderState { + next_rank: total.checked_sub(1), + completed: Vec::with_capacity(total), + }), + ready: std::sync::Condvar::new(), + } + } + + fn completed(&self) -> Vec { + self.state + .lock() + .expect("reverse-order state lock") + .completed + .clone() + } +} + +impl ClaimRelater for ReverseOrderRelater { + fn relate(&self, older: &str, newer: &str) -> Result { + let rank = *self + .ranks + .get(&(older.to_string(), newer.to_string())) + .expect("pair has a dispatch rank"); + // All representatives must be in flight before the highest rank is + // released. Each completion then unlocks exactly the preceding + // rank, forcing N-1..0 without sleeps or scheduler assumptions. + self.gate.wait(); + let mut state = self.state.lock().expect("reverse-order state lock"); + while state.next_rank != Some(rank) { + state = self.ready.wait(state).expect("reverse-order wait"); + } + state.completed.push(rank); + state.next_rank = rank.checked_sub(1); + self.ready.notify_all(); + drop(state); + let relation = if newer.contains("moved") { + ClaimRelation::Supersedes + } else { + ClaimRelation::Unrelated + }; + Ok(RelationVerdict { + relation, + score: 1.0, + }) + } + fn fingerprint(&self) -> String { + "reverse-order".to_owned() + } +} + +#[test] +fn parallel_reassembly_is_independent_of_completion_order() { + let (claims, embedder) = parallel_fanout_corpus(); + let seq = relate_claims_with_settled( + &claims, + &embedder, + &TextKeyedCountingRelater::new(), + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + ) + .expect("sequential"); + + let pending = prepare_pairs( + &claims, + &embedder, + th(0.9, 0.6), + &BTreeMap::new(), + &RelateTemporalPolicy::default(), + DEFAULT_CANDIDATE_PAIR_BUDGET, + CandidateCursor::start(), + ) + .expect("prepare") + .expect("non-empty candidates"); + let mut seen = BTreeSet::new(); + let ordered_pairs = pending + .pending + .iter() + .filter_map(|pair| { + let texts = ( + claims[pair.old_idx].1.text.clone(), + claims[pair.new_idx].1.text.clone(), + ); + seen.insert(texts.clone()).then_some(texts) + }) + .collect::>(); + let representative_calls = ordered_pairs.len(); + assert!( + representative_calls >= 2, + "need real fan-out to test ordering" + ); + + let relater = ReverseOrderRelater::new(ordered_pairs); + let par = relate_claims_settled_parallel( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &BTreeMap::new(), + Duration::MAX, + representative_calls, + ) + .expect("parallel"); + assert_eq!( + format!("{seq:?}"), + format!("{par:?}"), + "reassembly must not depend on worker completion order" + ); + assert_eq!( + relater.completed(), + (0..representative_calls).rev().collect::>(), + "test harness must actually force reverse completion order" + ); +} diff --git a/src/semantics/pipeline/tests/settlement.rs b/src/semantics/pipeline/tests/settlement.rs new file mode 100644 index 0000000..1464870 --- /dev/null +++ b/src/semantics/pipeline/tests/settlement.rs @@ -0,0 +1,210 @@ +use super::*; + +fn three_cluster_corpus() -> (Vec<(ClaimId, ClaimView)>, FixedEmbedder) { + // (distinct keyword, degrees) — no keyword is a substring of another. + let spec: [(&str, f32); 9] = [ + ("aardvark", 0.0), + ("abacus", 1.0), + ("acorn", 2.0), + ("baboon", 30.0), + ("badger", 31.0), + ("bagel", 32.0), + ("cactus", 60.0), + ("camel", 61.0), + ("candle", 62.0), + ]; + let table: Vec<(&'static str, Vec)> = spec + .iter() + .map(|(key, deg)| { + let rad = deg.to_radians(); + (*key, vec![rad.cos(), rad.sin()]) + }) + .collect(); + let claims: Vec<(ClaimId, ClaimView)> = spec + .iter() + .enumerate() + .map(|(idx, (key, _))| { + let seq = u64::try_from(idx).expect("small index") + 1; + let id = format!("claim_{idx:012x}"); + claim(&id, "x", &format!("the {key} subject"), seq) + }) + .collect(); + (claims, FixedEmbedder::new(table, 2)) +} + +const CORPUS_THRESHOLDS: RelateThresholds = RelateThresholds { + cluster: 0.98, + prefilter: 0.6, +}; + +#[test] +fn raw_candidate_page_bounds_total_pair_work_and_volume() { + let (claims, embedder) = three_cluster_corpus(); + let relater = CountingRelater::new(); + let out = relate_claims_settled_parallel_temporal( + &claims, + &embedder, + &relater, + CORPUS_THRESHOLDS, + &BTreeMap::new(), + ParallelRelateOptions { + temporal: &RelateTemporalPolicy::default(), + budget: Duration::MAX, + concurrency: 4, + candidate_pair_budget: 5, + candidate_cursor: CandidateCursor::start(), + }, + ) + .expect("bounded page"); + let partial = partial(&out).expect("nine claims require more than one page"); + assert_eq!(partial.candidate_pairs, 5); + assert_eq!(partial.next_candidate_cursor.offset(), 5); + assert!(partial.judgments.len() <= 5); + assert!(relater.count() <= 5); +} + +#[test] +fn clustering_is_deterministic_across_runs() { + let (claims, embedder) = three_cluster_corpus(); + let first = group_claims(&claims, &embedder, CORPUS_THRESHOLDS.cluster).expect("group"); + for _ in 0..5 { + let again = group_claims(&claims, &embedder, CORPUS_THRESHOLDS.cluster).expect("group"); + assert_eq!(first, again, "same input must yield identical clusters"); + } + // Stable ordering: members ascend within a group; groups are ordered by + // their first (smallest) member index — never hash-map iteration order. + for group in &first { + assert!(group.windows(2).all(|w| w[0] < w[1])); + } + let firsts: Vec = first.iter().map(|g| g[0]).collect(); + assert!(firsts.windows(2).all(|w| w[0] < w[1])); +} + +#[test] +fn relate_is_deterministic_across_runs() { + let (claims, embedder) = three_cluster_corpus(); + // Script a supersession and a conflict inside two different clusters so + // both output vectors are non-empty. + let relater = ScriptedRelater::new(vec![ + ("aardvark", "acorn", ClaimRelation::Supersedes), + ("baboon", "bagel", ClaimRelation::Conflict), + ]); + let first = relate_claims(&claims, &embedder, &relater, CORPUS_THRESHOLDS).expect("relate"); + assert_eq!(related(&first).expect("complete").supersessions.len(), 1); + assert_eq!(related(&first).expect("complete").conflicts.len(), 1); + for _ in 0..5 { + let again = relate_claims(&claims, &embedder, &relater, CORPUS_THRESHOLDS).expect("relate"); + assert_eq!( + related(&first).expect("complete").supersessions, + related(&again).expect("complete").supersessions, + "supersessions must be identical across runs" + ); + let ids = |out: &RelateOutcome| -> Vec { + related(out) + .expect("complete") + .conflicts + .iter() + .map(|c| c.conflict_id.to_string()) + .collect() + }; + assert_eq!( + ids(&first), + ids(&again), + "conflict ids must be identical across runs" + ); + } +} + +#[test] +fn unresolved_pair_taints_and_holds_dependent_supersession() { + struct PartiallyFailingRelater; + impl ClaimRelater for PartiallyFailingRelater { + fn relate(&self, older: &str, newer: &str) -> Result { + if older.contains("alpha") && newer.contains("beta") { + return Err(SemanticsError::Backend { + source: Box::new(std::io::Error::other("judge failed")), + }); + } + Ok(RelationVerdict { + relation: if older.contains("alpha") && newer.contains("gamma") { + ClaimRelation::Supersedes + } else { + ClaimRelation::Unrelated + }, + score: 1.0, + }) + } + + fn fingerprint(&self) -> String { + "partial".to_string() + } + } + + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "alpha old", 1), + claim("claim_bbbbbbbbbbbb", "x", "beta uncertain", 2), + claim("claim_cccccccccccc", "x", "gamma winner", 3), + ]; + let embedder = FixedEmbedder::new( + vec![ + ("alpha", vec![1.0, 0.0]), + ("beta", vec![1.0, 0.0]), + ("gamma", vec![1.0, 0.0]), + ], + 2, + ); + let outcome = relate_claims(&claims, &embedder, &PartiallyFailingRelater, th(0.9, 0.9)) + .expect("partial outcome"); + let outcome = partial(&outcome).expect("partial"); + assert_eq!(outcome.unresolved.len(), 1); + assert!(matches!( + outcome.held.as_slice(), + [HeldDecision::Supersession { old_claim, new_claim, .. }] + if old_claim.as_str() == "claim_aaaaaaaaaaaa" + && new_claim.as_str() == "claim_cccccccccccc" + )); +} + +#[test] +fn authoritative_pair_is_reused_without_judge_call() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "x", "alpha", 1), + claim("claim_bbbbbbbbbbbb", "x", "beta", 2), + claim("claim_cccccccccccc", "x", "gamma", 3), + ]; + let embedder = FixedEmbedder::new( + vec![ + ("alpha", vec![1.0, 0.0]), + ("beta", vec![1.0, 0.0]), + ("gamma", vec![1.0, 0.0]), + ], + 2, + ); + let relater = CountingRelater::new(); + let mut settled = BTreeMap::new(); + settled.insert( + (claims[0].0.clone(), claims[1].0.clone()), + RelationVerdict { + relation: ClaimRelation::Unrelated, + score: 1.0, + }, + ); + let outcome = relate_claims_with_settled( + &claims, + &embedder, + &relater, + th(0.9, 0.9), + &settled, + Duration::MAX, + ) + .expect("outcome"); + assert_eq!(relater.count(), 2); + assert_eq!( + outcome + .judgments() + .iter() + .filter(|judgment| judgment.reused_authority) + .count(), + 1 + ); +} diff --git a/src/semantics/pipeline/tests/temporal.rs b/src/semantics/pipeline/tests/temporal.rs new file mode 100644 index 0000000..4b619f8 --- /dev/null +++ b/src/semantics/pipeline/tests/temporal.rs @@ -0,0 +1,331 @@ +use super::*; + +#[test] +fn git_ancestry_orients_pairs_instead_of_ingest_sequence() { + let claims = vec![ + claim( + "claim_aaaaaaaaaaaa", + "storage", + "old storage uses postgres", + 20, + ), + claim( + "claim_bbbbbbbbbbbb", + "storage", + "new storage uses batpak", + 10, + ), + ]; + let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); + let relater = ScriptedRelater::new(vec![( + "old storage", + "new storage", + ClaimRelation::Supersedes, + )]); + let left = SourceSnapshotId::derive("ancestor"); + let right = SourceSnapshotId::derive("descendant"); + let mut temporal = RelateTemporalPolicy::default(); + temporal.bind_claim(&claims[0].0, &left); + temporal.bind_claim(&claims[1].0, &right); + temporal.insert_relation(&left, &right, TemporalRelation::Before); + + let out = relate_claims_with_settled_temporal( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &BTreeMap::new(), + &temporal, + Duration::MAX, + ) + .expect("relate"); + + assert_eq!( + related(&out).expect("complete").supersessions, + vec![( + claims[0].0.clone(), + claims[1].0.clone(), + "superseded by x.md:10".to_string(), + )] + ); + assert_eq!(out.judgments()[0].older_claim, claims[0].0); + assert_eq!(out.judgments()[0].newer_claim, claims[1].0); +} + +#[test] +fn incomparable_or_missing_source_order_never_calls_the_judge() { + for (relation, expected) in [ + ( + Some(TemporalRelation::Concurrent), + RelationFailureClass::TemporalConcurrent, + ), + (None, RelationFailureClass::TemporalUnknown), + ] { + let claims = vec![ + claim( + "claim_aaaaaaaaaaaa", + "storage", + "old storage uses postgres", + 1, + ), + claim( + "claim_bbbbbbbbbbbb", + "storage", + "new storage uses batpak", + 2, + ), + ]; + let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); + let relater = CountingRelater::new(); + let left = SourceSnapshotId::derive("left"); + let right = SourceSnapshotId::derive("right"); + let mut temporal = RelateTemporalPolicy::default(); + temporal.bind_claim(&claims[0].0, &left); + temporal.bind_claim(&claims[1].0, &right); + if let Some(relation) = relation { + temporal.insert_relation(&left, &right, relation); + } + + let out = relate_claims_settled_parallel_temporal( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &BTreeMap::new(), + ParallelRelateOptions { + temporal: &temporal, + budget: Duration::MAX, + concurrency: 4, + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + candidate_cursor: CandidateCursor::start(), + }, + ) + .expect("relate"); + + assert_eq!(relater.count(), 0); + let partial = partial(&out).expect("unresolved pair is partial"); + assert!(partial.judgments.is_empty()); + assert_eq!(partial.unresolved.len(), 1); + assert_eq!(partial.unresolved[0].failure.class, expected); + } +} + +#[test] +fn journal_authority_keeps_its_original_pair_direction() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "storage", "storage alpha", 1), + claim("claim_bbbbbbbbbbbb", "storage", "storage beta", 2), + ]; + let embedder = FixedEmbedder::new(vec![("storage", vec![1.0, 0.0])], 2); + let relater = CountingRelater::new(); + let left = SourceSnapshotId::derive("left"); + let right = SourceSnapshotId::derive("right"); + let mut temporal = RelateTemporalPolicy::default(); + temporal.bind_claim(&claims[0].0, &left); + temporal.bind_claim(&claims[1].0, &right); + temporal.insert_relation(&left, &right, TemporalRelation::After); + let mut settled = BTreeMap::new(); + settled.insert( + (claims[0].0.clone(), claims[1].0.clone()), + RelationVerdict { + relation: ClaimRelation::Unrelated, + score: 1.0, + }, + ); + + let out = relate_claims_settled_parallel_temporal( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &settled, + ParallelRelateOptions { + temporal: &temporal, + budget: Duration::MAX, + concurrency: 4, + candidate_pair_budget: DEFAULT_CANDIDATE_PAIR_BUDGET, + candidate_cursor: CandidateCursor::start(), + }, + ) + .expect("relate"); + + assert_eq!(relater.count(), 0); + assert_eq!(out.judgments()[0].older_claim, claims[0].0); + assert_eq!(out.judgments()[0].newer_claim, claims[1].0); + assert!(out.judgments()[0].reused_authority); +} + +#[test] +fn hard_candidate_budget_is_partial_bounded_and_resumable() { + let claims = (0_u64..5) + .map(|idx| { + claim( + &format!("claim_{idx:012x}"), + "shared", + &format!("shared candidate {idx}"), + idx + 1, + ) + }) + .collect::>(); + let embedder = FixedEmbedder::new(vec![("shared", vec![1.0, 0.0])], 2); + let mut settled = BTreeMap::new(); + let mut passes = 0_usize; + let mut cursor = CandidateCursor::start(); + let mut observed_cursors = Vec::new(); + loop { + let relater = CountingRelater::new(); + let out = relate_claims_settled_parallel_temporal( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &settled, + ParallelRelateOptions { + temporal: &RelateTemporalPolicy::default(), + budget: Duration::MAX, + concurrency: 2, + candidate_pair_budget: 2, + candidate_cursor: cursor, + }, + ) + .expect("bounded relate"); + assert!(relater.count() <= 2, "one pass cannot exceed the hard cap"); + for judgment in out.judgments() { + settled.insert( + (judgment.older_claim.clone(), judgment.newer_claim.clone()), + judgment.verdict, + ); + } + passes += 1; + match &out { + RelateOutcome::Complete(complete) => { + assert_eq!(complete.candidate_pairs, 10); + assert_eq!(settled.len(), 10, "all pairs settle after resumes"); + break; + } + RelateOutcome::Partial(partial) => { + assert!(partial.candidate_pairs <= 2); + assert_eq!(partial.candidate_pair_budget, 2); + cursor = partial.next_candidate_cursor; + observed_cursors.push(cursor.offset()); + assert!(passes < 10, "bounded passes must make progress"); + } + } + } + assert!(passes > 1); + assert_eq!(observed_cursors, vec![2, 4, 6, 8]); +} + +#[test] +fn resumed_completion_matches_one_shot_authority_edges() { + let claims = (0_u64..5) + .map(|idx| { + claim( + &format!("claim_{idx:012x}"), + "shared", + &format!("shared candidate {idx}"), + idx + 1, + ) + }) + .collect::>(); + let embedder = FixedEmbedder::new(vec![("shared", vec![1.0, 0.0])], 2); + let relater = ScriptedRelater::new(vec![ + ("candidate 0", "candidate 2", ClaimRelation::Supersedes), + ("candidate 3", "candidate 4", ClaimRelation::Conflict), + ]); + let one_shot = + relate_claims(&claims, &embedder, &relater, th(0.9, 0.6)).expect("one-shot relate"); + let one_shot = complete(&one_shot).expect("one-shot completion"); + + let mut settled = BTreeMap::new(); + let mut cursor = CandidateCursor::start(); + let paged = loop { + let outcome = relate_claims_settled_parallel_temporal( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &settled, + ParallelRelateOptions { + temporal: &RelateTemporalPolicy::default(), + budget: Duration::MAX, + concurrency: 1, + candidate_pair_budget: 2, + candidate_cursor: cursor, + }, + ) + .expect("paged relate"); + match outcome { + RelateOutcome::Complete(complete) => break complete, + RelateOutcome::Partial(partial) => { + for judgment in partial.judgments { + settled.insert( + (judgment.older_claim, judgment.newer_claim), + judgment.verdict, + ); + } + cursor = partial.next_candidate_cursor; + } + } + }; + + assert_eq!(paged.related.supersessions, one_shot.related.supersessions); + assert_eq!( + format!("{:?}", paged.related.conflicts), + format!("{:?}", one_shot.related.conflicts) + ); + let verdicts = |judgments: &[PairJudgment]| { + judgments + .iter() + .map(|judgment| { + ( + judgment.older_claim.clone(), + judgment.newer_claim.clone(), + judgment.verdict, + ) + }) + .collect::>() + }; + assert_eq!( + verdicts(&paged.related.judgments), + verdicts(&one_shot.related.judgments) + ); +} + +#[test] +fn changed_model_fingerprint_cannot_rejudge_journal_authority() { + let claims = vec![ + claim("claim_aaaaaaaaaaaa", "shared", "shared alpha", 1), + claim("claim_bbbbbbbbbbbb", "shared", "shared beta", 2), + ]; + let embedder = FixedEmbedder::new(vec![("shared", vec![1.0, 0.0])], 2); + let mut settled = BTreeMap::new(); + settled.insert( + (claims[0].0.clone(), claims[1].0.clone()), + RelationVerdict { + relation: ClaimRelation::Conflict, + score: 0.8, + }, + ); + for fingerprint in ["model-a:prompt-1", "model-b:prompt-99"] { + let relater = FingerprintRelater { + inner: CountingRelater::new(), + fingerprint, + }; + let out = relate_claims_with_settled( + &claims, + &embedder, + &relater, + th(0.9, 0.6), + &settled, + Duration::MAX, + ) + .expect("journal authority"); + assert_eq!(relater.inner.count(), 0); + assert!(out.judgments()[0].reused_authority); + assert_eq!( + out.judgments()[0].verdict, + settled[&(claims[0].0.clone(), claims[1].0.clone())] + ); + } +} diff --git a/src/semantics/score.rs b/src/semantics/score.rs new file mode 100644 index 0000000..f79ee4d --- /dev/null +++ b/src/semantics/score.rs @@ -0,0 +1,63 @@ +//! Checked fixed-point conversion at model boundaries. + +const PPM_SCALE: u32 = 1_000_000; + +/// Convert a unit-interval model score to integer parts per million. +/// +/// The conversion decomposes the IEEE-754 value into integer mantissa and +/// exponent components. That keeps the durable representation integer-only +/// without unchecked float-to-integer casts. +pub(crate) fn unit_interval_to_ppm(score: f32) -> u32 { + if score.is_nan() || score <= 0.0 { + return 0; + } + if score >= 1.0 { + return PPM_SCALE; + } + + let bits = score.to_bits(); + let raw_exponent = (bits >> 23) & 0xff; + if raw_exponent == 0 { + return 0; + } + let exponent = i32::try_from(raw_exponent).unwrap_or(0) - 127; + let shift = 23_i32.saturating_sub(exponent); + let Ok(shift) = u32::try_from(shift) else { + return 0; + }; + let Some(denominator) = 1_u64.checked_shl(shift) else { + return 0; + }; + let mantissa = u64::from((bits & 0x7f_ffff) | (1 << 23)); + let numerator = mantissa.saturating_mul(u64::from(PPM_SCALE)); + let rounded = numerator + .saturating_add(denominator / 2) + .checked_div(denominator) + .unwrap_or(0); + u32::try_from(rounded).unwrap_or(PPM_SCALE).min(PPM_SCALE) +} + +/// Convert integer parts per million to a unit-interval model score. +pub(crate) fn ppm_to_unit_interval(ppm: u32) -> f32 { + let ppm = ppm.min(PPM_SCALE); + let thousands = u16::try_from(ppm / 1_000).unwrap_or(1_000); + let remainder = u16::try_from(ppm % 1_000).unwrap_or(0); + (f32::from(thousands) * 1_000.0 + f32::from(remainder)) / 1_000_000.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_point_conversion_clamps_and_rounds() { + assert_eq!(unit_interval_to_ppm(f32::NAN), 0); + assert_eq!(unit_interval_to_ppm(-1.0), 0); + assert_eq!(unit_interval_to_ppm(0.0), 0); + assert_eq!(unit_interval_to_ppm(0.5), 500_000); + assert_eq!(unit_interval_to_ppm(1.0), 1_000_000); + assert_eq!(unit_interval_to_ppm(2.0), 1_000_000); + assert!((ppm_to_unit_interval(500_000) - 0.5).abs() < f32::EPSILON); + assert!((ppm_to_unit_interval(2_000_000) - 1.0).abs() < f32::EPSILON); + } +} diff --git a/src/semantics/traits.rs b/src/semantics/traits.rs index c587515..cf8b1b4 100644 --- a/src/semantics/traits.rs +++ b/src/semantics/traits.rs @@ -144,6 +144,7 @@ pub trait Proposer { /// rather than dividing by zero or indexing out of bounds. A returned `0.0` /// therefore means "no usable signal," which is the safe neutral value for /// downstream thresholding. +#[must_use] pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { if a.is_empty() || a.len() != b.len() { return 0.0; diff --git a/src/surfaces/bootstrap.rs b/src/surfaces/bootstrap.rs index 9508d6d..2cfc715 100644 --- a/src/surfaces/bootstrap.rs +++ b/src/surfaces/bootstrap.rs @@ -1,11 +1,17 @@ //! First-run workspace bootstrap for the memory-agent surface. use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use crate::config::{SemanticsConfig, TexoRootConfig, WorkspaceEntry}; use crate::error::TexoError; +mod model; +mod resolver; + +pub use model::{BootstrapDecision, BootstrapInputs}; +pub use resolver::resolve_bootstrap; + /// Environment variable pointing at an extractor binary. pub const ENV_EXTRACT_BIN: &str = "TEXO_EXTRACT_BIN"; /// Environment variable selecting the record-once extractor cache directory. @@ -17,66 +23,6 @@ pub const DEFAULT_EXTRACT_CACHE: &str = ".texo/extract-cache"; /// Whether first-run bootstrap may point `extractor_cmd` at `texo extract`. pub const EXTRACT_SUBCOMMAND_READY: bool = true; -/// Inputs to extractor resolution. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BootstrapInputs { - /// `TEXO_EXTRACT_BIN` value, when present. - pub extract_bin: Option, - /// Resolved neutral model API-key value, when present. - pub model_api_key: Option, - /// Current executable path. - pub current_exe: PathBuf, -} - -/// Extractor resolution result. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BootstrapDecision { - /// Shell command written to config, or `None` for heuristic extraction. - pub extractor_cmd: Option, - /// Whether `[semantics] enabled = true` should be written. - pub semantics_enabled: bool, - /// Optional startup warning. - pub warning: Option, -} - -/// Resolve bootstrap extraction settings without reading process environment. -#[must_use] -pub fn resolve_bootstrap(root: &Path, inputs: &BootstrapInputs) -> BootstrapDecision { - if let Some(raw) = &inputs.extract_bin { - if raw.trim().is_empty() { - return BootstrapDecision { - extractor_cmd: None, - semantics_enabled: false, - warning: None, - }; - } - let cmd = extractor_cmd_for(root, raw); - return BootstrapDecision { - extractor_cmd: Some(cmd), - semantics_enabled: true, - warning: None, - }; - } - let has_key = inputs - .model_api_key - .as_deref() - .is_some_and(|key| !key.trim().is_empty()); - if has_key && EXTRACT_SUBCOMMAND_READY { - let exe = inputs.current_exe.to_string_lossy(); - return BootstrapDecision { - extractor_cmd: Some(extractor_cmd_for(root, &exe)), - semantics_enabled: true, - warning: None, - }; - } - BootstrapDecision { - extractor_cmd: None, - semantics_enabled: false, - warning: (!has_key) - .then(|| "TEXO_LLM_API_KEY is not set; using heuristic session extraction".to_string()), - } -} - /// Resolve bootstrap extraction settings from process environment. /// /// # Errors @@ -164,6 +110,8 @@ pub(crate) fn prospective_config( #[cfg(test)] mod tests { + use std::path::PathBuf; + use super::*; fn inputs(bin: Option<&str>, key: Option<&str>) -> BootstrapInputs { diff --git a/src/surfaces/bootstrap/model.rs b/src/surfaces/bootstrap/model.rs new file mode 100644 index 0000000..fc3d7d4 --- /dev/null +++ b/src/surfaces/bootstrap/model.rs @@ -0,0 +1,25 @@ +//! Bootstrap input and decision shapes. + +use std::path::PathBuf; + +/// Inputs to extractor resolution. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootstrapInputs { + /// `TEXO_EXTRACT_BIN` value, when present. + pub extract_bin: Option, + /// Resolved neutral model API-key value, when present. + pub model_api_key: Option, + /// Current executable path. + pub current_exe: PathBuf, +} + +/// Extractor resolution result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootstrapDecision { + /// Shell command written to config, or `None` for heuristic extraction. + pub extractor_cmd: Option, + /// Whether `[semantics] enabled = true` should be written. + pub semantics_enabled: bool, + /// Optional startup warning. + pub warning: Option, +} diff --git a/src/surfaces/bootstrap/resolver.rs b/src/surfaces/bootstrap/resolver.rs new file mode 100644 index 0000000..1c72445 --- /dev/null +++ b/src/surfaces/bootstrap/resolver.rs @@ -0,0 +1,43 @@ +//! Pure bootstrap resolution policy. + +use std::path::Path; + +use super::{extractor_cmd_for, BootstrapDecision, BootstrapInputs, EXTRACT_SUBCOMMAND_READY}; + +/// Resolve bootstrap extraction settings without reading process environment. +#[must_use] +pub fn resolve_bootstrap(root: &Path, inputs: &BootstrapInputs) -> BootstrapDecision { + if let Some(raw) = &inputs.extract_bin { + if raw.trim().is_empty() { + return BootstrapDecision { + extractor_cmd: None, + semantics_enabled: false, + warning: None, + }; + } + let cmd = extractor_cmd_for(root, raw); + return BootstrapDecision { + extractor_cmd: Some(cmd), + semantics_enabled: true, + warning: None, + }; + } + let has_key = inputs + .model_api_key + .as_deref() + .is_some_and(|key| !key.trim().is_empty()); + if has_key && EXTRACT_SUBCOMMAND_READY { + let exe = inputs.current_exe.to_string_lossy(); + return BootstrapDecision { + extractor_cmd: Some(extractor_cmd_for(root, &exe)), + semantics_enabled: true, + warning: None, + }; + } + BootstrapDecision { + extractor_cmd: None, + semantics_enabled: false, + warning: (!has_key) + .then(|| "TEXO_LLM_API_KEY is not set; using heuristic session extraction".to_string()), + } +} diff --git a/src/surfaces/cli/dispatch/durability.rs b/src/surfaces/cli/dispatch/durability.rs new file mode 100644 index 0000000..0aa22ef --- /dev/null +++ b/src/surfaces/cli/dispatch/durability.rs @@ -0,0 +1,115 @@ +//! Backup and replication commands. + +use std::process::ExitCode; + +use crate::config::TexoRootConfig; +use crate::error::TexoError; + +use super::super::{ + follow_replica_until_shutdown, observed_at_ms, render, BackupCmd, DispatchContext, ReplicaCmd, +}; + +pub(super) fn backup(cli: &DispatchContext, command: BackupCmd) -> Result { + match command { + BackupCmd::Create { dest, json } => { + let config_path = cli.root.join(".texo/config.toml"); + let root_config = + TexoRootConfig::load(&config_path).map_err(|error| TexoError::Config { + detail: error.to_string(), + source: Some(Box::new(error)), + })?; + let workspace = root_config + .resolve(cli.workspace.as_deref()) + .map_err(|error| TexoError::Config { + detail: error.to_string(), + source: Some(Box::new(error)), + })?; + let store = crate::host::open_workspace_store(&cli.root, &workspace.workspace_id)?; + let report = crate::backup::create( + &cli.root, + &workspace, + store.as_ref(), + &dest, + observed_at_ms(), + )?; + let output = serde_json::to_value(report)?; + if json { + render::json(&output)?; + } else { + render::backup(&output)?; + } + Ok(ExitCode::SUCCESS) + } + BackupCmd::Verify { + dest, + expect_manifest_hash, + json, + } => { + let report = crate::backup::verify_with_expected_manifest_hash( + &dest, + expect_manifest_hash.as_deref(), + )?; + let verified = report.verified; + let output = serde_json::to_value(report)?; + if json { + render::json(&output)?; + } else { + render::backup(&output)?; + } + Ok(if verified { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + }) + } + BackupCmd::Restore { + source, + expect_manifest_hash, + json, + } => { + let report = + crate::backup::restore(&source, &cli.root, expect_manifest_hash.as_deref())?; + let output = serde_json::to_value(report)?; + if json { + render::json(&output)?; + } else { + render::backup(&output)?; + } + Ok(ExitCode::SUCCESS) + } + } +} + +pub(super) fn replica(cli: &DispatchContext, command: ReplicaCmd) -> Result { + let report = match command { + ReplicaCmd::Bootstrap { replica, json } => { + let _ = json; + crate::replication::bootstrap(&cli.root, cli.workspace.as_deref(), &replica)? + } + ReplicaCmd::Follow { + replica, + json, + watch: false, + interval_ms, + } => { + let _ = (json, interval_ms); + crate::replication::follow_once(&cli.root, cli.workspace.as_deref(), &replica)? + } + ReplicaCmd::Follow { + replica, + json, + watch: true, + interval_ms, + } => { + let _ = json; + return follow_replica_until_shutdown( + &cli.root, + cli.workspace.as_deref(), + &replica, + interval_ms, + ); + } + }; + render::json(&serde_json::to_value(report)?)?; + Ok(ExitCode::SUCCESS) +} diff --git a/src/surfaces/cli/dispatch/integrations.rs b/src/surfaces/cli/dispatch/integrations.rs new file mode 100644 index 0000000..97fe606 --- /dev/null +++ b/src/surfaces/cli/dispatch/integrations.rs @@ -0,0 +1,115 @@ +//! Installation, hook, and diagnostic commands. + +use std::io::Read; +use std::process::ExitCode; + +use serde_json::json; + +use crate::error::TexoError; +use crate::install::ClientTarget; + +use super::super::{open_host, render, DispatchContext, HookCmd}; + +pub(super) fn install( + cli: &DispatchContext, + client: &[ClientTarget], + dry_run: bool, + json_output: bool, +) -> Result { + let workspace = cli.workspace.as_deref().unwrap_or("demo"); + let report = crate::install::install_for_journal( + &cli.root, + workspace, + cli.journal.as_deref(), + client, + dry_run, + )?; + let output = serde_json::to_value(report)?; + if json_output { + render::json(&output)?; + } else { + render::installation(&output)?; + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn uninstall( + cli: &DispatchContext, + client: &[ClientTarget], + dry_run: bool, + json_output: bool, +) -> Result { + let report = crate::install::uninstall(&cli.root, client, dry_run)?; + let output = serde_json::to_value(report)?; + if json_output { + render::json(&output)?; + } else { + render::installation(&output)?; + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn hook(cli: &DispatchContext, command: &HookCmd) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let (event, data, json_output) = match command { + HookCmd::SessionStart { json } => ( + "session_start", + host.invoke_json( + "texo.context.agent", + &json!({ + "subject": null, + "include_stale": true, + "allow_unsettled": true + }), + )?, + *json, + ), + HookCmd::FilesChanged { json } => { + let mut bytes = Vec::new(); + std::io::stdin() + .take((crate::hooks::MAX_INPUT_BYTES + 1) as u64) + .read_to_end(&mut bytes)?; + let input = crate::hooks::parse_files_changed(&bytes)?; + let mut reports = Vec::with_capacity(input.paths.len()); + for path in input.paths { + reports.push(host.invoke_json("texo.staleness.check", &json!({"path": path}))?); + } + ("files_changed", json!({"reports": reports}), *json) + } + HookCmd::PreCommit { json } => ( + "pre_commit", + host.invoke_json("texo.verify.run", &json!({}))?, + *json, + ), + }; + let output = json!({ + "schema": "texo.hook-result.v1", + "event": event, + "advisory": true, + "data": data + }); + let _ = json_output; + render::json(&output)?; + Ok(ExitCode::SUCCESS) +} + +pub(super) fn doctor( + cli: &DispatchContext, + deep: bool, + fix: bool, + json_output: bool, +) -> Result { + let report = crate::doctor::diagnose(&cli.root, cli.workspace.as_deref(), deep, fix); + let broken = report.status == crate::doctor::DoctorStatus::Broken; + let output = serde_json::to_value(report)?; + if json_output { + render::json(&output)?; + } else { + render::doctor(&output)?; + } + Ok(if broken { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + }) +} diff --git a/src/surfaces/cli/dispatch/mod.rs b/src/surfaces/cli/dispatch/mod.rs new file mode 100644 index 0000000..517a5ac --- /dev/null +++ b/src/surfaces/cli/dispatch/mod.rs @@ -0,0 +1,103 @@ +//! Exhaustive CLI command routing. + +use std::process::ExitCode; + +use crate::error::TexoError; + +use super::{Command, DispatchContext}; + +mod durability; +mod integrations; +mod semantics; +mod surfaces; +mod workspace; + +pub(super) fn route(cli: &DispatchContext, command: Command) -> Result { + match command { + Command::Init { workspace } => workspace::init(cli, &workspace), + Command::Ingest { + path, + dry_run, + strict, + json, + } => workspace::ingest(cli, &path, dry_run, strict, json), + Command::Claims { subject, json } => workspace::claims(cli, subject.as_deref(), json), + Command::Supersede { + old, + new, + reason, + decided_by, + json, + } => workspace::supersede(cli, &old, &new, &reason, &decided_by, json), + Command::CheckStaleness { path, json } => workspace::check_staleness(cli, &path, json), + Command::AgentContext { + subject, + out, + json, + allow_unsettled, + } => workspace::agent_context(cli, subject.as_deref(), out, json, allow_unsettled), + Command::Compile { + out, + allow_unsettled, + } => workspace::compile(cli, &out, allow_unsettled), + Command::Relate { + json, + strict, + pair_budget, + candidate_cursor, + rejudge_pair, + } => semantics::relate( + cli, + json, + strict, + pair_budget, + candidate_cursor, + rejudge_pair.as_deref(), + ), + Command::Conflicts { json, commit } => workspace::conflicts(cli, json, commit), + Command::Verify { json } => workspace::verify(cli, json), + Command::Stats { json } => workspace::stats(cli, json), + Command::Index { + scip, + max_files, + max_file_bytes, + max_total_bytes, + json, + } => semantics::index( + cli, + scip.as_deref(), + max_files, + max_file_bytes, + max_total_bytes, + json, + ), + Command::Reconcile { + max_per_claim, + max_candidates, + min_score_ppm, + json, + } => semantics::reconcile(cli, max_per_claim, max_candidates, min_score_ppm, json), + Command::Mcp => surfaces::mcp(cli), + Command::Serve(options) => { + surfaces::serve(cli, options.with_default_journal(cli.journal.as_deref())) + } + Command::Extract { path } => Ok(surfaces::extract(&path)), + Command::Session { cmd } => surfaces::session(cli, cmd), + Command::Host { cmd } => surfaces::host(cli, &cmd), + Command::Ops { cmd } => surfaces::ops(cmd), + Command::Install { + client, + dry_run, + json, + } => integrations::install(cli, &client, dry_run, json), + Command::Uninstall { + client, + dry_run, + json, + } => integrations::uninstall(cli, &client, dry_run, json), + Command::Hook { cmd } => integrations::hook(cli, &cmd), + Command::Doctor { deep, fix, json } => integrations::doctor(cli, deep, fix, json), + Command::Backup { cmd } => durability::backup(cli, cmd), + Command::Replica { cmd } => durability::replica(cli, cmd), + } +} diff --git a/src/surfaces/cli/dispatch/semantics.rs b/src/surfaces/cli/dispatch/semantics.rs new file mode 100644 index 0000000..4bbc370 --- /dev/null +++ b/src/surfaces/cli/dispatch/semantics.rs @@ -0,0 +1,121 @@ +//! Code-intelligence and semantic-settlement commands. + +use std::path::Path; +use std::process::ExitCode; + +use serde_json::{json, Value}; + +use crate::error::TexoError; + +use super::super::{observed_at_ms, open_host, render, DispatchContext}; + +pub(super) fn index( + cli: &DispatchContext, + scip: Option<&Path>, + max_files: Option, + max_file_bytes: Option, + max_total_bytes: Option, + json_output: bool, +) -> Result { + let _ = json_output; + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let observed_at_ms = observed_at_ms(); + let source = host.invoke_json( + "texo.knowledge.index", + &json!({ + "observed_at_ms": observed_at_ms, + "max_files": max_files, + "max_file_bytes": max_file_bytes, + "max_total_bytes": max_total_bytes + }), + )?; + let code = host.invoke_json( + "texo.code.index.build", + &json!({ + "snapshot_id": source.get("snapshot_id").cloned(), + "scip_path": scip, + "observed_at_ms": observed_at_ms + }), + )?; + let output = json!({ + "schema": "texo.index.v2", + "source": source, + "code": code + }); + let truncated = ["source", "code"].iter().any(|phase| { + output + .get(*phase) + .and_then(|value| value.get("coverage")) + .and_then(|coverage| coverage.get("truncated")) + .and_then(Value::as_bool) + .unwrap_or(false) + }); + render::json(&output)?; + Ok(if truncated { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + }) +} + +pub(super) fn reconcile( + cli: &DispatchContext, + max_per_claim: Option, + max_candidates: Option, + min_score_ppm: Option, + json_output: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.knowledge.reconcile", + &json!({ + "observed_at_ms": observed_at_ms(), + "max_per_claim": max_per_claim, + "max_candidates": max_candidates, + "min_score_ppm": min_score_ppm, + "budget_secs": null, + "concurrency": null + }), + )?; + if json_output { + render::json(&output)?; + } else { + render::reconcile(&output)?; + } + Ok(partial_exit(&output)) +} + +pub(super) fn relate( + cli: &DispatchContext, + json_output: bool, + strict: bool, + pair_budget: Option, + candidate_cursor: Option, + rejudge_pair: Option<&[String]>, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.relate.run", + &json!({ + "observed_at_ms": observed_at_ms(), + "strict": strict, + "max_candidate_pairs": pair_budget, + "candidate_cursor": candidate_cursor, + "rejudge_pair": rejudge_pair + }), + )?; + if json_output { + render::json(&output)?; + } else { + render::relate(&output)?; + } + Ok(partial_exit(&output)) +} + +fn partial_exit(output: &Value) -> ExitCode { + if output.get("outcome").and_then(Value::as_str) == Some("partial") { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + } +} diff --git a/src/surfaces/cli/dispatch/surfaces.rs b/src/surfaces/cli/dispatch/surfaces.rs new file mode 100644 index 0000000..2fb2268 --- /dev/null +++ b/src/surfaces/cli/dispatch/surfaces.rs @@ -0,0 +1,88 @@ +//! Host, protocol, and discovery surface commands. + +use std::path::Path; +use std::process::ExitCode; + +use serde_json::{json, Value}; + +use crate::error::TexoError; + +use super::super::{ + extract as run_extract, open_host, refresh_selected_reader, render, serve as run_server, + DispatchContext, HostCmd, OpsCmd, ServeOptions, SessionCmd, +}; + +pub(super) fn host(cli: &DispatchContext, command: &HostCmd) -> Result { + match command { + HostCmd::Fingerprint => { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json("texo.host.fingerprint", &json!({}))?; + render::json(&output)?; + Ok(ExitCode::SUCCESS) + } + } +} + +pub(super) fn mcp(cli: &DispatchContext) -> Result { + refresh_selected_reader(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + crate::surfaces::mcp_stdio::run(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + Ok(ExitCode::SUCCESS) +} + +pub(super) fn serve(cli: &DispatchContext, options: ServeOptions) -> Result { + run_server(options, &cli.root, cli.workspace.as_deref()) +} + +pub(super) fn extract(path: &Path) -> ExitCode { + run_extract(path) +} + +pub(super) fn session(cli: &DispatchContext, command: SessionCmd) -> Result { + match command { + SessionCmd::Export { session_id } => { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = + host.invoke_json("texo.session.export", &json!({"session_id": session_id}))?; + render::session_markdown( + output + .get("markdown") + .and_then(Value::as_str) + .unwrap_or_default(), + )?; + Ok(ExitCode::SUCCESS) + } + } +} + +pub(super) fn ops(command: OpsCmd) -> Result { + let inventory = crate::agent_catalog::operation_inventory(); + match command { + OpsCmd::List { json } => { + if json { + render::json(&inventory)?; + } else { + render::operations(&inventory)?; + } + } + OpsCmd::Describe { name, json } => { + let operation = inventory["operations"] + .as_array() + .and_then(|operations| { + operations + .iter() + .find(|operation| operation["name"] == name) + }) + .cloned() + .ok_or_else(|| TexoError::OpInput { + op: "texo ops describe".to_string(), + detail: format!("unknown operation `{name}`"), + })?; + if json { + render::json(&operation)?; + } else { + render::operations(&json!({"operations": [operation]}))?; + } + } + } + Ok(ExitCode::SUCCESS) +} diff --git a/src/surfaces/cli/dispatch/workspace.rs b/src/surfaces/cli/dispatch/workspace.rs new file mode 100644 index 0000000..a2fc6bf --- /dev/null +++ b/src/surfaces/cli/dispatch/workspace.rs @@ -0,0 +1,238 @@ +//! Workspace, projection, and inspection commands. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use serde_json::{json, Value}; + +use crate::error::TexoError; +use crate::host::TexoHost; + +use super::super::{observed_at_ms, open_host, render, DispatchContext}; + +pub(super) fn init(cli: &DispatchContext, workspace: &str) -> Result { + let mut host = + TexoHost::open_for_init(cli.root.clone(), workspace.to_string(), observed_at_ms())?; + let output = host.invoke_json("texo.workspace.init", &json!({ "workspace_id": workspace }))?; + render::init(&cli.root, &output)?; + Ok(ExitCode::SUCCESS) +} + +pub(super) fn ingest( + cli: &DispatchContext, + path: &Path, + dry_run: bool, + strict: bool, + json_output: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.ingest.run", + &json!({ + "path": path, + "dry_run": dry_run, + "strict": strict, + "observed_at_ms": observed_at_ms() + }), + )?; + if json_output { + render::json(&output)?; + } else { + render::ingest(&output)?; + } + Ok(partial_exit(&output)) +} + +pub(super) fn claims( + cli: &DispatchContext, + subject: Option<&str>, + json_output: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json("texo.claims.list", &json!({"subject": subject}))?; + if json_output { + let claims = output + .get("claims") + .cloned() + .unwrap_or(Value::Array(Vec::new())); + render::json(&claims)?; + } else { + render::claims(&output)?; + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn supersede( + cli: &DispatchContext, + old: &str, + new: &str, + reason: &str, + decided_by: &str, + json_output: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.claim.supersede", + &json!({ + "old": old, + "new": new, + "reason": reason, + "decided_by": decided_by, + "observed_at_ms": observed_at_ms() + }), + )?; + if json_output { + render::json(&output)?; + } else { + render::supersede(&output)?; + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn check_staleness( + cli: &DispatchContext, + path: &Path, + json_output: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json("texo.staleness.check", &json!({"path": path}))?; + let has_findings = output + .get("diagnostics") + .and_then(Value::as_array) + .is_some_and(|diagnostics| !diagnostics.is_empty()); + if json_output { + render::json(&output)?; + } else { + render::staleness(&output)?; + } + Ok(if has_findings { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + }) +} + +pub(super) fn agent_context( + cli: &DispatchContext, + subject: Option<&str>, + out: Option, + json_output: bool, + allow_unsettled: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.context.agent", + &json!({ + "subject": subject, + "include_stale": true, + "allow_unsettled": allow_unsettled + }), + )?; + let rendered = serde_json::to_string_pretty(&output)?; + if let Some(path) = out { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &rendered)?; + if json_output { + render::json(&output)?; + } + } else { + let _ = json_output; + render::json(&output)?; + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn compile( + cli: &DispatchContext, + out: &Path, + allow_unsettled: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json( + "texo.compile.run", + &json!({ + "out_dir": out, + "observed_at_ms": observed_at_ms(), + "allow_unsettled": allow_unsettled + }), + )?; + render::compile(out, &output)?; + Ok(ExitCode::SUCCESS) +} + +pub(super) fn conflicts( + cli: &DispatchContext, + json_output: bool, + commit: bool, +) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + if commit { + let output = host.invoke_json( + "texo.conflicts.commit", + &json!({"observed_at_ms": observed_at_ms()}), + )?; + if json_output { + render::json(&output)?; + } else { + render::conflicts_committed(&output)?; + } + } else { + let output = host.invoke_json("texo.conflicts.list", &json!({}))?; + if json_output { + render::json(&output)?; + } else { + render::conflicts(&output)?; + } + } + Ok(ExitCode::SUCCESS) +} + +pub(super) fn verify(cli: &DispatchContext, json_output: bool) -> Result { + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json("texo.verify.run", &json!({}))?; + let failed = ["projection_ok", "journal_ok", "transitions_ok"] + .iter() + .any(|key| output.get(*key).and_then(Value::as_bool) == Some(false)); + if json_output { + render::json(&output)?; + } else if failed { + return Err(TexoError::Verify { + failures: output + .get("errors") + .and_then(Value::as_array) + .map(|errors| { + errors + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect::>() + }) + .unwrap_or_default(), + }); + } else { + render::verify(&output)?; + } + Ok(if failed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + }) +} + +pub(super) fn stats(cli: &DispatchContext, json_output: bool) -> Result { + let _ = json_output; + let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; + let output = host.invoke_json("texo.stats.read", &json!({}))?; + render::json(&output)?; + Ok(ExitCode::SUCCESS) +} + +fn partial_exit(output: &Value) -> ExitCode { + if output.get("outcome").and_then(Value::as_str) == Some("partial") { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + } +} diff --git a/src/surfaces/cli/mod.rs b/src/surfaces/cli/mod.rs index 76b6e62..6fb2f0d 100644 --- a/src/surfaces/cli/mod.rs +++ b/src/surfaces/cli/mod.rs @@ -1,22 +1,21 @@ //! Clap-driven CLI surface. -use std::io::Read; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::time::{SystemTime, UNIX_EPOCH}; -use clap::{Parser, Subcommand}; -use serde_json::{json, Value}; - use crate::config::TexoRootConfig; use crate::error::TexoError; use crate::host::TexoHost; use crate::install::ClientTarget; +use clap::{Args, Parser, Subcommand}; /// CLI render helpers. pub mod render; +mod dispatch; + #[derive(Parser)] #[command(name = "texo", about = "Agent-ready claim memory in one local binary")] struct Cli { @@ -86,17 +85,17 @@ enum Command { out: Option, #[arg(long)] json: bool, - /// Refuse context while semantic settlement remains incomplete. + /// Permit context before semantic settlement completes. #[arg(long)] - strict_settlement: bool, + allow_unsettled: bool, }, /// Compile human-readable artifacts. Compile { #[arg(long, default_value = "public")] out: PathBuf, - /// Refuse compilation while semantic settlement remains incomplete. + /// Permit artifact compilation before semantic settlement completes. #[arg(long)] - strict_settlement: bool, + allow_unsettled: bool, }, /// Semantic supersession + conflict pass (needs `TEXO_LLM_API_KEY`). Relate { @@ -105,6 +104,15 @@ enum Command { /// Refuse derived authority while any required pair is unresolved. #[arg(long)] strict: bool, + /// Hard ceiling for previously-unsettled candidate pairs in this pass. + #[arg(long)] + pair_budget: Option, + /// Resume candidate enumeration after this durable cursor. + #[arg(long = "pair-cursor")] + candidate_cursor: Option, + /// Evict and freshly judge one settled pair; first judgment remains authoritative. + #[arg(long, num_args = 2, value_names = ["OLDER_CLAIM", "NEWER_CLAIM"])] + rejudge_pair: Option>, }, /// Report possible conflicts. Conflicts { @@ -160,26 +168,7 @@ enum Command { /// Run MCP stdio server. Mcp, /// Run the memory-agent HTTP server. - Serve { - /// Listen address. - #[arg(long)] - addr: Option, - /// Workspace root. - #[arg(long)] - root: Option, - /// Workspace id. - #[arg(long)] - workspace: Option, - /// Physical journal id. - #[arg(long)] - journal: Option, - /// Optional private-network canonical replica-source listener address. - #[arg(long)] - replica_addr: Option, - /// Environment variable containing the replica MAC secret. - #[arg(long, default_value = "TEXO_REPLICA_TOKEN")] - replica_token_env: String, - }, + Serve(ServeOptions), /// Extract claims from one path. Extract { path: PathBuf }, /// Session utilities. @@ -350,15 +339,45 @@ enum ReplicaCmd { }, } +#[derive(Args)] struct ServeOptions { + /// Listen address. + #[arg(long)] addr: Option, + /// Workspace root. + #[arg(long)] root: Option, + /// Workspace id. + #[arg(long)] workspace: Option, + /// Physical journal id. + #[arg(long)] journal: Option, + /// Optional private-network canonical replica-source listener address. + #[arg(long)] replica_addr: Option, + /// Environment variable containing the replica MAC secret. + #[arg(long, default_value = "TEXO_REPLICA_TOKEN")] replica_token_env: String, } +impl ServeOptions { + #[must_use] + fn with_default_journal(mut self, default_journal: Option<&str>) -> Self { + if self.journal.is_none() { + self.journal = default_journal.map(str::to_string); + } + self + } +} + +struct PreparedServe { + listener: TcpListener, + config: crate::surfaces::http::server::ServerConfig, + shutdown: crate::surfaces::http::server::ShutdownHandle, + replica_thread: Option>>, +} + /// Run the CLI and return the requested process exit code. /// /// # Errors @@ -369,588 +388,25 @@ pub fn run() -> Result { dispatch(cli) } -#[expect( - clippy::too_many_lines, - reason = "CLI dispatch table mirrors the command surface during rebuild" -)] fn dispatch(cli: Cli) -> Result { - match cli.command { - Command::Init { workspace } => { - let mut host = - TexoHost::open_for_init(cli.root.clone(), workspace.clone(), observed_at_ms())?; - let output = - host.invoke_json("texo.workspace.init", &json!({ "workspace_id": workspace }))?; - render::init(&cli.root, &output); - Ok(ExitCode::SUCCESS) - } - Command::Ingest { - path, - dry_run, - strict, - json, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json( - "texo.ingest.run", - &json!({ - "path": path, - "dry_run": dry_run, - "strict": strict, - "observed_at_ms": observed_at_ms() - }), - )?; - if json { - render::json(&output)?; - } else { - render::ingest(&output); - } - Ok( - if output.get("outcome").and_then(Value::as_str) == Some("partial") { - ExitCode::from(2) - } else { - ExitCode::SUCCESS - }, - ) - } - Command::Claims { subject, json } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json("texo.claims.list", &json!({"subject": subject}))?; - if json { - let claims = output - .get("claims") - .cloned() - .unwrap_or(Value::Array(Vec::new())); - render::json(&claims)?; - } else { - render::claims(&output); - } - Ok(ExitCode::SUCCESS) - } - Command::Supersede { - old, - new, - reason, - decided_by, - json, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json( - "texo.claim.supersede", - &json!({ - "old": old, - "new": new, - "reason": reason, - "decided_by": decided_by, - "observed_at_ms": observed_at_ms() - }), - )?; - if json { - render::json(&output)?; - } else { - render::supersede(&output); - } - Ok(ExitCode::SUCCESS) - } - Command::CheckStaleness { path, json } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json("texo.staleness.check", &json!({"path": path}))?; - let has_findings = output - .get("diagnostics") - .and_then(Value::as_array) - .is_some_and(|diagnostics| !diagnostics.is_empty()); - if json { - render::json(&output)?; - } else { - render::staleness(&output); - } - Ok(if has_findings { - ExitCode::FAILURE - } else { - ExitCode::SUCCESS - }) - } - Command::AgentContext { - subject, - out, - json, - strict_settlement, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json( - "texo.context.agent", - &json!({ - "subject": subject, - "include_stale": true, - "strict_settlement": strict_settlement - }), - )?; - let rendered = serde_json::to_string_pretty(&output)?; - if let Some(path) = out { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&path, &rendered)?; - if json { - render::json(&output)?; - } - } else { - let _ = json; - render::json(&output)?; - } - Ok(ExitCode::SUCCESS) - } - Command::Compile { - out, - strict_settlement, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let out_for_input = out.clone(); - let output = host.invoke_json( - "texo.compile.run", - &json!({ - "out_dir": out_for_input, - "observed_at_ms": observed_at_ms(), - "strict_settlement": strict_settlement - }), - )?; - render::compile(&out, &output); - Ok(ExitCode::SUCCESS) - } - Command::Conflicts { json, commit } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - if commit { - let output = host.invoke_json( - "texo.conflicts.commit", - &json!({"observed_at_ms": observed_at_ms()}), - )?; - if json { - render::json(&output)?; - } else { - render::conflicts_committed(&output); - } - } else { - let output = host.invoke_json("texo.conflicts.list", &json!({}))?; - if json { - render::json(&output)?; - } else { - render::conflicts(&output); - } - } - Ok(ExitCode::SUCCESS) - } - Command::Verify { json } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json("texo.verify.run", &json!({}))?; - let failed = ["projection_ok", "journal_ok", "transitions_ok"] - .iter() - .any(|key| output.get(*key).and_then(Value::as_bool) == Some(false)); - if json { - render::json(&output)?; - } else if failed { - return Err(TexoError::Verify { - failures: output - .get("errors") - .and_then(Value::as_array) - .map(|errors| { - errors - .iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect::>() - }) - .unwrap_or_default(), - }); - } else { - render::verify(&output); - } - Ok(if failed { - ExitCode::FAILURE - } else { - ExitCode::SUCCESS - }) - } - Command::Stats { json: _ } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json("texo.stats.read", &json!({}))?; - render::json(&output)?; - Ok(ExitCode::SUCCESS) - } - Command::Index { - scip, - max_files, - max_file_bytes, - max_total_bytes, - json: _, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let observed_at_ms = observed_at_ms(); - let source = host.invoke_json( - "texo.knowledge.index", - &json!({ - "observed_at_ms": observed_at_ms, - "max_files": max_files, - "max_file_bytes": max_file_bytes, - "max_total_bytes": max_total_bytes - }), - )?; - let code = host.invoke_json( - "texo.code.index.build", - &json!({ - "snapshot_id": source.get("snapshot_id").cloned(), - "scip_path": scip, - "observed_at_ms": observed_at_ms - }), - )?; - let output = json!({ - "schema": "texo.index.v2", - "source": source, - "code": code - }); - // Incomplete capture must not read as success: mirror the partial exit - // used by the other arms when either phase reports truncated coverage. - let truncated = ["source", "code"].iter().any(|phase| { - output - .get(*phase) - .and_then(|value| value.get("coverage")) - .and_then(|coverage| coverage.get("truncated")) - .and_then(Value::as_bool) - .unwrap_or(false) - }); - render::json(&output)?; - if truncated { - Ok(ExitCode::from(2)) - } else { - Ok(ExitCode::SUCCESS) - } - } - Command::Reconcile { - max_per_claim, - max_candidates, - min_score_ppm, - json, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json( - "texo.knowledge.reconcile", - &json!({ - "observed_at_ms": observed_at_ms(), - "max_per_claim": max_per_claim, - "max_candidates": max_candidates, - "min_score_ppm": min_score_ppm, - "budget_secs": null, - "concurrency": null - }), - )?; - if json { - render::json(&output)?; - } else { - render::reconcile(&output); - } - Ok( - if output.get("outcome").and_then(Value::as_str) == Some("partial") { - ExitCode::from(2) - } else { - ExitCode::SUCCESS - }, - ) - } - Command::Host { - cmd: HostCmd::Fingerprint, - } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json("texo.host.fingerprint", &json!({}))?; - render::json(&output)?; - Ok(ExitCode::SUCCESS) - } - Command::Relate { json, strict } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = host.invoke_json( - "texo.relate.run", - &json!({"observed_at_ms": observed_at_ms(), "strict": strict}), - )?; - if json { - render::json(&output)?; - } else { - render::relate(&output); - } - Ok( - if output.get("outcome").and_then(Value::as_str) == Some("partial") { - ExitCode::from(2) - } else { - ExitCode::SUCCESS - }, - ) - } - Command::Mcp => { - refresh_selected_reader(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - crate::surfaces::mcp_stdio::run( - &cli.root, - cli.workspace.as_deref(), - cli.journal.as_deref(), - )?; - Ok(ExitCode::SUCCESS) - } - Command::Serve { - addr, - root, - workspace, - journal, - replica_addr, - replica_token_env, - } => serve( - ServeOptions { - addr, - root, - workspace, - journal: journal.or_else(|| cli.journal.clone()), - replica_addr, - replica_token_env, - }, - &cli.root, - cli.workspace.as_deref(), - ), - Command::Extract { path } => Ok(extract(&path)), - Command::Session { cmd } => match cmd { - SessionCmd::Export { session_id } => { - let mut host = - open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let output = - host.invoke_json("texo.session.export", &json!({"session_id": session_id}))?; - render::session_markdown( - output - .get("markdown") - .and_then(Value::as_str) - .unwrap_or_default(), - ); - Ok(ExitCode::SUCCESS) - } - }, - Command::Ops { cmd } => { - let inventory = crate::agent_catalog::operation_inventory(); - match cmd { - OpsCmd::List { json } => { - if json { - render::json(&inventory)?; - } else { - render::operations(&inventory); - } - } - OpsCmd::Describe { name, json } => { - let operation = inventory["operations"] - .as_array() - .and_then(|operations| { - operations - .iter() - .find(|operation| operation["name"] == name) - }) - .cloned() - .ok_or_else(|| TexoError::OpInput { - op: "texo ops describe".to_string(), - detail: format!("unknown operation `{name}`"), - })?; - if json { - render::json(&operation)?; - } else { - render::operations(&json!({"operations": [operation]})); - } - } - } - Ok(ExitCode::SUCCESS) - } - Command::Install { - client, - dry_run, - json, - } => { - let workspace = cli.workspace.as_deref().unwrap_or("demo"); - let report = crate::install::install_for_journal( - &cli.root, - workspace, - cli.journal.as_deref(), - &client, - dry_run, - )?; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::installation(&output); - } - Ok(ExitCode::SUCCESS) - } - Command::Uninstall { - client, - dry_run, - json, - } => { - let report = crate::install::uninstall(&cli.root, &client, dry_run)?; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::installation(&output); - } - Ok(ExitCode::SUCCESS) - } - Command::Hook { cmd } => { - let mut host = open_host(&cli.root, cli.workspace.as_deref(), cli.journal.as_deref())?; - let (event, data, json) = match cmd { - HookCmd::SessionStart { json } => ( - "session_start", - host.invoke_json( - "texo.context.agent", - &json!({ - "subject": null, - "include_stale": true, - "strict_settlement": false - }), - )?, - json, - ), - HookCmd::FilesChanged { json } => { - let mut bytes = Vec::new(); - std::io::stdin() - .take((crate::hooks::MAX_INPUT_BYTES + 1) as u64) - .read_to_end(&mut bytes)?; - let input = crate::hooks::parse_files_changed(&bytes)?; - let mut reports = Vec::with_capacity(input.paths.len()); - for path in input.paths { - reports.push( - host.invoke_json("texo.staleness.check", &json!({"path": path}))?, - ); - } - ("files_changed", json!({"reports": reports}), json) - } - HookCmd::PreCommit { json } => ( - "pre_commit", - host.invoke_json("texo.verify.run", &json!({}))?, - json, - ), - }; - let output = json!({ - "schema": "texo.hook-result.v1", - "event": event, - "advisory": true, - "data": data - }); - let _ = json; - render::json(&output)?; - Ok(ExitCode::SUCCESS) - } - Command::Doctor { deep, fix, json } => { - let report = crate::doctor::diagnose(&cli.root, cli.workspace.as_deref(), deep, fix); - let broken = report.status == crate::doctor::DoctorStatus::Broken; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::doctor(&output); - } - Ok(if broken { - ExitCode::FAILURE - } else { - ExitCode::SUCCESS - }) - } - Command::Backup { cmd } => match cmd { - BackupCmd::Create { dest, json } => { - let config_path = cli.root.join(".texo/config.toml"); - let root_config = - TexoRootConfig::load(&config_path).map_err(|error| TexoError::Config { - detail: error.to_string(), - source: Some(Box::new(error)), - })?; - let workspace = root_config - .resolve(cli.workspace.as_deref()) - .map_err(|error| TexoError::Config { - detail: error.to_string(), - source: Some(Box::new(error)), - })?; - let store = crate::host::open_workspace_store(&cli.root, &workspace.workspace_id)?; - let report = crate::backup::create( - &cli.root, - &workspace, - store.as_ref(), - &dest, - observed_at_ms(), - )?; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::backup(&output); - } - Ok(ExitCode::SUCCESS) - } - BackupCmd::Verify { - dest, - expect_manifest_hash, - json, - } => { - let report = crate::backup::verify_with_expected_manifest_hash( - &dest, - expect_manifest_hash.as_deref(), - )?; - let verified = report.verified; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::backup(&output); - } - Ok(if verified { - ExitCode::SUCCESS - } else { - ExitCode::FAILURE - }) - } - BackupCmd::Restore { - source, - expect_manifest_hash, - json, - } => { - let report = - crate::backup::restore(&source, &cli.root, expect_manifest_hash.as_deref())?; - let output = serde_json::to_value(report)?; - if json { - render::json(&output)?; - } else { - render::backup(&output); - } - Ok(ExitCode::SUCCESS) - } - }, - Command::Replica { cmd } => { - let report = match cmd { - ReplicaCmd::Bootstrap { replica, json: _ } => { - crate::replication::bootstrap(&cli.root, cli.workspace.as_deref(), &replica)? - } - ReplicaCmd::Follow { - replica, - json: _, - watch: false, - interval_ms: _, - } => { - crate::replication::follow_once(&cli.root, cli.workspace.as_deref(), &replica)? - } - ReplicaCmd::Follow { - replica, - json: _, - watch: true, - interval_ms, - } => { - return follow_replica_until_shutdown( - &cli.root, - cli.workspace.as_deref(), - &replica, - interval_ms, - ); - } - }; - render::json(&serde_json::to_value(report)?)?; - Ok(ExitCode::SUCCESS) - } - } + let Cli { + root, + workspace, + journal, + command, + } = cli; + let context = DispatchContext { + root, + workspace, + journal, + }; + dispatch::route(&context, command) +} + +struct DispatchContext { + root: PathBuf, + workspace: Option, + journal: Option, } fn follow_replica_until_shutdown( @@ -984,6 +440,21 @@ fn serve( global_root: &Path, global_workspace: Option<&str>, ) -> Result { + let PreparedServe { + listener, + config, + shutdown, + replica_thread, + } = prepare_serve(options, global_root, global_workspace)?; + let replica_stats = run_serve(&listener, config, &shutdown, replica_thread)?; + Ok(report_serve(replica_stats)) +} + +fn prepare_serve( + options: ServeOptions, + global_root: &Path, + global_workspace: Option<&str>, +) -> Result { let addr = options .addr .or_else(|| std::env::var("TEXO_AGENT_ADDR").ok()) @@ -1006,7 +477,7 @@ fn serve( .unwrap_or_else(|| "memory".to_string()); let decision = crate::surfaces::bootstrap::resolve_bootstrap_from_env(&root)?; if let Some(warning) = &decision.warning { - render::serve_warning(warning); + render::serve_warning(warning)?; } crate::surfaces::bootstrap::ensure_workspace(&root, &workspace, &decision)?; let root_config = crate::config::TexoRootConfig::load(&root.join(".texo/config.toml")) @@ -1035,7 +506,7 @@ fn serve( which: crate::error::SurfaceKind::Http, detail: error.to_string(), })?; - render::serve_listening(local); + render::serve_listening(local)?; let store = crate::host::open_workspace_journal_store(&root, &workspace, selected_journal.id.as_str())?; let gateway = root_config.gateway; @@ -1051,7 +522,7 @@ fn serve( journal_id: selected_journal.id.to_string(), store: Some(store.clone()), projection_cache: std::sync::Arc::new(std::sync::Mutex::new(None)), - chat_enabled: crate::host::grants_model_capability(Some(chat_role.api_key)), + chat_enabled: crate::host::grants_model_capability(Some(chat_role.api_key.as_str())), }; let config = crate::surfaces::http::server::ServerConfig::new(local.to_string(), state); let shutdown = crate::surfaces::http::server::ShutdownHandle::new(); @@ -1064,7 +535,21 @@ fn serve( &served_workspace, &shutdown, )?; - let http_result = crate::surfaces::http::server::serve_listener(listener, config, &shutdown); + Ok(PreparedServe { + listener, + config, + shutdown, + replica_thread, + }) +} + +fn run_serve( + listener: &TcpListener, + config: crate::surfaces::http::server::ServerConfig, + shutdown: &crate::surfaces::http::server::ShutdownHandle, + replica_thread: Option>>, +) -> Result, TexoError> { + let http_result = crate::surfaces::http::server::serve_listener(listener, config, shutdown); shutdown.shutdown(); let replica_result = replica_thread .map(|thread| { @@ -1075,7 +560,11 @@ fn serve( }) .transpose()?; let _http_stats = http_result?; - if let Some(stats) = replica_result { + Ok(replica_result) +} + +fn report_serve(replica_stats: Option) -> ExitCode { + if let Some(stats) = replica_stats { tracing::debug!( accepted = stats.accepted_connections, served = stats.served_requests, @@ -1083,7 +572,7 @@ fn serve( "replica listener stopped" ); } - Ok(ExitCode::SUCCESS) + ExitCode::SUCCESS } fn refresh_selected_reader( @@ -1166,7 +655,7 @@ fn extract(path: &Path) -> ExitCode { match extract_impl(path) { Ok(()) => ExitCode::SUCCESS, Err(error) => { - render::extract_error(error.as_ref()); + let _rendered = render::extract_error(error.as_ref()); ExitCode::FAILURE } } diff --git a/src/surfaces/cli/render.rs b/src/surfaces/cli/render.rs index 2be66f9..ec13104 100644 --- a/src/surfaces/cli/render.rs +++ b/src/surfaces/cli/render.rs @@ -1,29 +1,37 @@ //! CLI render helpers. +use std::io::{self, Write as _}; + use serde_json::Value; use crate::error::TexoError; /// Render one typed CLI failure with its causal chain and recovery facts. -#[expect(clippy::print_stderr, reason = "CLI error contract")] -pub fn cli_error(error: &TexoError) { +/// +/// # Errors +/// Returns an I/O error when standard error cannot be written. +pub fn cli_error(error: &TexoError) -> io::Result<()> { use std::error::Error as _; - eprintln!("error[{}]: {error}", error.code()); + let stderr = io::stderr(); + let mut out = stderr.lock(); + writeln!(out, "error[{}]: {error}", error.code())?; let mut source = error.source(); while let Some(cause) = source { - eprintln!("caused by: {cause}"); + writeln!(out, "caused by: {cause}")?; source = cause.source(); } let facts = error.facts(); - eprintln!("committed: {}", facts.committed); - eprintln!( + writeln!(out, "committed: {}", facts.committed)?; + writeln!( + out, "retry: {}", if facts.retry_safe { "safe" } else { "unsafe" } - ); + )?; if let Some(resume) = facts.resume { - eprintln!("resume: {resume}"); + writeln!(out, "resume: {resume}")?; } + Ok(()) } /// Print a JSON value unchanged except for pretty formatting. @@ -31,15 +39,19 @@ pub fn cli_error(error: &TexoError) { /// # Errors /// /// Returns [`TexoError::Json`] when the value cannot be serialized. -#[expect(clippy::print_stdout, reason = "CLI output contract")] pub fn json(value: &Value) -> Result<(), TexoError> { - println!("{}", serde_json::to_string_pretty(value)?); + let stdout = io::stdout(); + writeln!(stdout.lock(), "{}", serde_json::to_string_pretty(value)?)?; Ok(()) } /// Print the operation catalog in one stable line per operation. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn operations(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn operations(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); let operations = value .get("operations") .and_then(Value::as_array) @@ -57,32 +69,40 @@ pub fn operations(value: &Value) { .get("agent_tool") .and_then(Value::as_str) .map_or("human", |tool| tool); - println!("{name}\t{effect}\t{agent}"); + writeln!(out, "{name}\t{effect}\t{agent}")?; } + Ok(()) } /// Print the init message. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn init(root: &std::path::Path, value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn init(root: &std::path::Path, value: &Value) -> Result<(), TexoError> { let workspace = value .get("workspace_id") .and_then(Value::as_str) .unwrap_or("demo"); - println!( + writeln!( + io::stdout().lock(), "Initialized texo workspace '{}' at {}/.texo", workspace, root.display() - ); + )?; + Ok(()) } /// Print ingest summary. -#[expect( - clippy::print_stdout, - clippy::print_stderr, - reason = "CLI output and warning contract" -)] -pub fn ingest(value: &Value) { - println!( +/// +/// # Errors +/// Returns [`TexoError::Io`] when an output stream cannot be written. +pub fn ingest(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let stderr = io::stderr(); + let mut out = stdout.lock(); + let mut warnings = stderr.lock(); + writeln!( + out, "ingested {} sources, {} claims ({})", value .get("sources_observed") @@ -96,26 +116,35 @@ pub fn ingest(value: &Value) { .get("workspace_id") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; if value.get("empty").and_then(Value::as_bool) == Some(true) { - eprintln!("warning: source root exists but contains no markdown sources"); + writeln!( + warnings, + "warning: source root exists but contains no markdown sources" + )?; } if let Some(skipped) = value.get("skipped").and_then(Value::as_array) { for row in skipped { - eprintln!( + writeln!( + warnings, "warning: skipped {} ({})", row.get("path").and_then(Value::as_str).unwrap_or("unknown"), row.get("code") .and_then(Value::as_str) .unwrap_or("source.io") - ); + )?; } } + Ok(()) } /// Print claims in the old four-line block format. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn claims(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn claims(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); let claims = value .get("claims") .and_then(Value::as_array) @@ -133,16 +162,18 @@ pub fn claims(value: &Value) { .get("subject_hint") .and_then(Value::as_str) .unwrap_or_default(); - println!("{id} {} {subject}", status_label(status)); - println!( + writeln!(out, "{id} {} {subject}", status_label(status))?; + writeln!( + out, " \"{}\"", claim .get("text") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; let source = claim.get("source").unwrap_or(&Value::Null); - println!( + writeln!( + out, " source: {}:{}", source .get("path") @@ -152,20 +183,23 @@ pub fn claims(value: &Value) { .get("line_start") .and_then(Value::as_u64) .unwrap_or(0) - ); + )?; let receipt = claim.get("receipt").unwrap_or(&Value::Null); - println!( + writeln!( + out, " seq: {}", receipt.get("sequence").and_then(Value::as_u64).unwrap_or(0) - ); - println!( + )?; + writeln!( + out, " receipt: {}", receipt .get("event_id") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; } + Ok(()) } fn status_label(value: &str) -> &'static str { @@ -178,17 +212,23 @@ fn status_label(value: &str) -> &'static str { } /// Print supersession summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn supersede(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn supersede(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); if value.get("already_applied").and_then(Value::as_bool) == Some(true) { - println!( + writeln!( + out, "claim {} already superseded by {} (no-op)", value.get("old").and_then(Value::as_str).unwrap_or_default(), value.get("new").and_then(Value::as_str).unwrap_or_default() - ); - return; + )?; + return Ok(()); } - println!( + writeln!( + out, "superseded {} with {} at local seq {}", value.get("old").and_then(Value::as_str).unwrap_or_default(), value.get("new").and_then(Value::as_str).unwrap_or_default(), @@ -197,56 +237,74 @@ pub fn supersede(value: &Value) { .and_then(|receipt| receipt.get("global_sequence")) .and_then(Value::as_u64) .unwrap_or(0) - ); + )?; + Ok(()) } /// Print staleness diagnostics. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn staleness(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn staleness(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); let diagnostics = value .get("diagnostics") .and_then(Value::as_array) .map_or(&[] as &[Value], Vec::as_slice); for diag in diagnostics { - println!( + writeln!( + out, "{}:{} warning — {}", diag.get("file").and_then(Value::as_str).unwrap_or_default(), diag.get("line_start").and_then(Value::as_u64).unwrap_or(0), diag.get("message") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; } if diagnostics.is_empty() { - println!("no stale claims detected"); + writeln!(out, "no stale claims detected")?; } + Ok(()) } /// Print compile summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn compile(out: &std::path::Path, value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn compile(out: &std::path::Path, value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); let files = value .get("files") .and_then(Value::as_array) .map_or(&[] as &[Value], Vec::as_slice); for file in files { - println!( + writeln!( + stdout, "wrote {}/{}", out.display(), file.as_str().unwrap_or_default() - ); + )?; } + Ok(()) } /// Print conflicts. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn conflicts(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn conflicts(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); let open = value .get("open") .and_then(Value::as_array) .map_or(&[] as &[Value], Vec::as_slice); for entry in open { - println!( + writeln!( + out, "{} {} vs {} ({})", entry .get("conflict_id") @@ -264,53 +322,75 @@ pub fn conflicts(value: &Value) { .get("subject_hint") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; } + Ok(()) } /// Print conflict commit summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn conflicts_committed(value: &Value) { - println!( +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn conflicts_committed(value: &Value) -> Result<(), TexoError> { + writeln!( + io::stdout().lock(), "committed {} conflicts", value.as_array().map_or(0, Vec::len) - ); + )?; + Ok(()) } /// Print verify summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn verify(value: &Value) { - println!( +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn verify(value: &Value) -> Result<(), TexoError> { + writeln!( + io::stdout().lock(), "ok — replayed through local seq {}", value .get("replayed_through_sequence") .or_else(|| value.get("frontier")) .and_then(Value::as_u64) .unwrap_or(0) - ); + )?; + Ok(()) } /// Print session export markdown. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn session_markdown(markdown: &str) { - println!("{markdown}"); +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn session_markdown(markdown: &str) -> Result<(), TexoError> { + writeln!(io::stdout().lock(), "{markdown}")?; + Ok(()) } /// Print serve startup line. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn serve_listening(addr: std::net::SocketAddr) { - println!("texo-agent listening on http://{addr}"); +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn serve_listening(addr: std::net::SocketAddr) -> Result<(), TexoError> { + writeln!(io::stdout().lock(), "texo-agent listening on http://{addr}")?; + Ok(()) } /// Print serve bootstrap warning. -#[expect(clippy::print_stderr, reason = "CLI output contract")] -pub fn serve_warning(message: &str) { - eprintln!("{message}"); +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard error cannot be written. +pub fn serve_warning(message: &str) -> Result<(), TexoError> { + writeln!(io::stderr().lock(), "{message}")?; + Ok(()) } /// Print a concise install or uninstall change report. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn installation(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn installation(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); let verb = if value.get("workspace_id").is_some() { "install" } else { @@ -320,14 +400,19 @@ pub fn installation(value: &Value) { .get("dry_run") .and_then(Value::as_bool) .unwrap_or(false); - println!("texo {verb}{}", if dry_run { " (dry run)" } else { "" }); + writeln!( + out, + "texo {verb}{}", + if dry_run { " (dry run)" } else { "" } + )?; for change in value .get("changes") .and_then(Value::as_array) .into_iter() .flatten() { - println!( + writeln!( + out, " {:<9} {}", change .get("action") @@ -337,27 +422,34 @@ pub fn installation(value: &Value) { .get("path") .and_then(Value::as_str) .unwrap_or("unknown") - ); + )?; } + Ok(()) } /// Print a concise doctor report with repair guidance. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn doctor(value: &Value) { - println!( +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn doctor(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); + writeln!( + out, "texo doctor: {}", value .get("status") .and_then(Value::as_str) .unwrap_or("broken") - ); + )?; for check in value .get("checks") .and_then(Value::as_array) .into_iter() .flatten() { - println!( + writeln!( + out, " {:<5} {:<24} {}", check .get("status") @@ -368,32 +460,39 @@ pub fn doctor(value: &Value) { .get("detail") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; if let Some(fix) = check.get("fix").and_then(Value::as_str) { - println!(" fix: {fix}"); + writeln!(out, " fix: {fix}")?; } } + Ok(()) } /// Print a concise backup create or verification report. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn backup(value: &Value) { +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn backup(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let mut out = stdout.lock(); if let Some(verified) = value.get("verified").and_then(Value::as_bool) { - println!( + writeln!( + out, "backup {}: {}", value .get("dest") .and_then(Value::as_str) .unwrap_or_default(), if verified { "verified" } else { "INVALID" } - ); + )?; for finding in value .get("findings") .and_then(Value::as_array) .into_iter() .flatten() { - println!( + writeln!( + out, " {}: {}", finding .get("kind") @@ -403,10 +502,11 @@ pub fn backup(value: &Value) { .get("detail") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; } } else if value.get("chain_verified").and_then(Value::as_bool) == Some(true) { - println!( + writeln!( + out, "backup restored: {} ({} files, {} bytes; chain verified)", value .get("dest") @@ -420,9 +520,10 @@ pub fn backup(value: &Value) { .get("store_bytes") .and_then(Value::as_u64) .unwrap_or(0) - ); + )?; } else { - println!( + writeln!( + out, "backup created: {} ({} files, {} bytes)", value .get("dest") @@ -436,34 +537,66 @@ pub fn backup(value: &Value) { .get("store_bytes") .and_then(Value::as_u64) .unwrap_or(0) - ); - println!( + )?; + writeln!( + out, "manifest hash: {} (store this outside the backup)", value .get("manifest_hash_hex") .and_then(Value::as_str) .unwrap_or_default() - ); + )?; } + Ok(()) } /// Print an extractor error with the extractor subcommand prefix. -#[expect(clippy::print_stderr, reason = "CLI output contract")] -pub fn extract_error(error: &dyn std::error::Error) { - eprint!("texo extract: {error}"); +/// +/// # Errors +/// Returns an I/O error when standard error cannot be written. +pub fn extract_error(error: &dyn std::error::Error) -> io::Result<()> { + let stderr = io::stderr(); + let mut out = stderr.lock(); + write!(out, "texo extract: {error}")?; let mut source = error.source(); while let Some(cause) = source { - eprint!(": {cause}"); + write!(out, ": {cause}")?; source = cause.source(); } - eprintln!(); + writeln!(out) } /// Print relate summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn relate(value: &Value) { - println!( - "related {} claims: {} supersessions, {} conflicts", +/// +/// # Errors +/// Returns [`TexoError::Io`] when an output stream cannot be written. +pub fn relate(value: &Value) -> Result<(), TexoError> { + let stdout = io::stdout(); + let stderr = io::stderr(); + write_relate(&mut stdout.lock(), &mut stderr.lock(), value)?; + Ok(()) +} + +fn write_relate( + out: &mut dyn io::Write, + warnings: &mut dyn io::Write, + value: &Value, +) -> io::Result<()> { + let outcome = value + .get("outcome") + .and_then(Value::as_str) + .unwrap_or("partial"); + let candidate_pairs = value + .get("candidate_pairs") + .and_then(Value::as_u64) + .unwrap_or(0); + let candidate_pair_budget = value + .get("candidate_pair_budget") + .and_then(Value::as_u64) + .unwrap_or(0); + writeln!( + out, + "relate {outcome}: {} claims; supersessions: {}; conflicts: {}", value .get("claims_related") .and_then(Value::as_u64) @@ -476,13 +609,49 @@ pub fn relate(value: &Value) { .get("conflicts") .and_then(Value::as_array) .map_or(0, Vec::len) - ); + )?; + writeln!( + out, + "candidate pairs: {candidate_pairs} (page budget: {candidate_pair_budget})" + )?; + if outcome == "partial" { + if let Some(cursor) = value.get("next_candidate_cursor").and_then(Value::as_u64) { + writeln!(out, "resume candidate cursor: {cursor}")?; + } + } + if let Some(pair) = value.get("rejudged_pair").filter(|pair| !pair.is_null()) { + writeln!( + out, + "rejudged pair {} -> {}: {} -> {} (first judgment remains authoritative)", + pair.get("older_claim") + .and_then(Value::as_str) + .unwrap_or("unknown"), + pair.get("newer_claim") + .and_then(Value::as_str) + .unwrap_or("unknown"), + pair.get("prior_relation") + .and_then(Value::as_str) + .unwrap_or("unknown"), + pair.get("fresh_relation") + .and_then(Value::as_str) + .unwrap_or("unknown") + )?; + } + if let Some(rows) = value.get("warnings").and_then(Value::as_array) { + for warning in rows.iter().filter_map(Value::as_str) { + writeln!(warnings, "warning: {warning}")?; + } + } + Ok(()) } /// Print semantic claim↔code reconciliation summary. -#[expect(clippy::print_stdout, reason = "CLI output contract")] -pub fn reconcile(value: &Value) { - println!( +/// +/// # Errors +/// Returns [`TexoError::Io`] when standard output cannot be written. +pub fn reconcile(value: &Value) -> Result<(), TexoError> { + writeln!( + io::stdout().lock(), "reconciliation {}: {} accepted, {} rejected, {} unresolved, {} already linked", value .get("outcome") @@ -501,5 +670,9 @@ pub fn reconcile(value: &Value) { .get("already_linked") .and_then(Value::as_u64) .unwrap_or(0), - ); + )?; + Ok(()) } + +#[cfg(test)] +mod tests; diff --git a/src/surfaces/cli/render/tests.rs b/src/surfaces/cli/render/tests.rs new file mode 100644 index 0000000..d45c3d6 --- /dev/null +++ b/src/surfaces/cli/render/tests.rs @@ -0,0 +1,84 @@ +use super::write_relate; +use serde_json::json; + +type TestResult = Result<(), Box>; + +fn rendered(value: &serde_json::Value) -> Result<(String, String), Box> { + let mut out = Vec::new(); + let mut warnings = Vec::new(); + write_relate(&mut out, &mut warnings, value)?; + Ok((String::from_utf8(out)?, String::from_utf8(warnings)?)) +} + +#[test] +fn relate_complete_reports_outcome_and_page_budget() -> TestResult { + let value = json!({ + "outcome": "complete", + "claims_related": 3, + "supersessions": [{}], + "conflicts": [{}, {}], + "candidate_pairs": 3, + "candidate_pair_budget": 4096, + "warnings": [] + }); + let (out, warnings) = rendered(&value)?; + + assert_eq!( + out, + "relate complete: 3 claims; supersessions: 1; conflicts: 2\n\ +candidate pairs: 3 (page budget: 4096)\n" + ); + assert!(warnings.is_empty()); + Ok(()) +} + +#[test] +fn relate_partial_reports_cursor_and_warnings() -> TestResult { + let value = json!({ + "outcome": "partial", + "claims_related": 5, + "supersessions": [], + "conflicts": [], + "candidate_pairs": 2, + "candidate_pair_budget": 2, + "next_candidate_cursor": 4, + "warnings": ["candidate page is incomplete"] + }); + let (out, warnings) = rendered(&value)?; + + assert!(out.contains("relate partial")); + assert!(out.contains("page budget: 2")); + assert!(out.contains("resume candidate cursor: 4")); + assert_eq!(warnings, "warning: candidate page is incomplete\n"); + Ok(()) +} + +#[test] +fn relate_rejudge_reports_transition_without_cache_provenance() -> TestResult { + let value = json!({ + "outcome": "complete", + "claims_related": 2, + "supersessions": [], + "conflicts": [], + "candidate_pairs": 1, + "candidate_pair_budget": 10, + "rejudged_pair": { + "older_claim": "claim_aaaaaaaaaaaa", + "newer_claim": "claim_bbbbbbbbbbbb", + "prior_relation": "supersedes", + "fresh_relation": "conflict", + "judge_fingerprint": "provider-secret-adjacent", + "cache_key": "private-cache-key" + } + }); + let (out, warnings) = rendered(&value)?; + + assert!(out.contains( + "rejudged pair claim_aaaaaaaaaaaa -> claim_bbbbbbbbbbbb: supersedes -> conflict" + )); + assert!(out.contains("first judgment remains authoritative")); + assert!(!out.contains("provider-secret-adjacent")); + assert!(!out.contains("private-cache-key")); + assert!(warnings.is_empty()); + Ok(()) +} diff --git a/src/surfaces/http/chunked.rs b/src/surfaces/http/chunked.rs index bcfe9e2..0521a6e 100644 --- a/src/surfaces/http/chunked.rs +++ b/src/surfaces/http/chunked.rs @@ -1,110 +1,13 @@ //! HTTP chunked transfer decoding. -use std::io::{BufRead, ErrorKind, Read}; - -use crate::surfaces::http::client::HttpClientError; - -/// Decode an HTTP chunked response body. -/// -/// # Errors -/// -/// Returns [`HttpClientError::MalformedResponse`] for malformed framing, -/// [`HttpClientError::BodyTooLarge`] when the decoded body exceeds `body_cap`, -/// or [`HttpClientError::Io`] when the reader fails. -pub fn decode_chunked( - reader: &mut R, - body_cap: usize, -) -> Result, HttpClientError> { - let mut body = Vec::new(); - loop { - let line = read_line(reader)?; - let size_text = line - .split_once(';') - .map_or(line.as_str(), |(head, _)| head) - .trim(); - if size_text.is_empty() { - return Err(HttpClientError::MalformedResponse { - detail: "empty chunk size".to_string(), - }); - } - let size = usize::from_str_radix(size_text, 16).map_err(|_| { - HttpClientError::MalformedResponse { - detail: "invalid chunk size".to_string(), - } - })?; - if size == 0 { - consume_trailers(reader)?; - return Ok(body); - } - if body.len().saturating_add(size) > body_cap { - return Err(HttpClientError::BodyTooLarge); - } - let start = body.len(); - body.resize(start + size, 0); - read_exact_chunk(reader, &mut body[start..], "chunk data")?; - let mut crlf = [0_u8; 2]; - read_exact_chunk(reader, &mut crlf, "chunk delimiter")?; - if crlf != *b"\r\n" { - return Err(HttpClientError::MalformedResponse { - detail: "chunk data was not followed by CRLF".to_string(), - }); - } - } -} - -fn read_exact_chunk( - reader: &mut R, - buffer: &mut [u8], - during: &'static str, -) -> Result<(), HttpClientError> { - reader.read_exact(buffer).map_err(|source| { - if source.kind() == ErrorKind::UnexpectedEof { - HttpClientError::MalformedResponse { - detail: format!("truncated {during}"), - } - } else { - HttpClientError::Io { during, source } - } - }) -} - -fn consume_trailers(reader: &mut R) -> Result<(), HttpClientError> { - loop { - let line = read_line(reader)?; - if line.is_empty() { - return Ok(()); - } - } -} - -fn read_line(reader: &mut R) -> Result { - let mut bytes = Vec::new(); - let read = reader - .read_until(b'\n', &mut bytes) - .map_err(|source| HttpClientError::Io { - during: "chunk line", - source, - })?; - if read == 0 { - return Err(HttpClientError::MalformedResponse { - detail: "truncated chunked body".to_string(), - }); - } - if !bytes.ends_with(b"\r\n") { - return Err(HttpClientError::MalformedResponse { - detail: "chunk line missing CRLF".to_string(), - }); - } - bytes.truncate(bytes.len() - 2); - String::from_utf8(bytes).map_err(|_| HttpClientError::MalformedResponse { - detail: "chunk line was not UTF-8".to_string(), - }) -} +pub use super::codec::decode_chunked; #[cfg(test)] mod tests { use std::io::Cursor; + use crate::surfaces::http::client::HttpClientError; + use super::*; #[test] diff --git a/src/surfaces/http/client.rs b/src/surfaces/http/client.rs index 0d425d8..4d8e41a 100644 --- a/src/surfaces/http/client.rs +++ b/src/surfaces/http/client.rs @@ -682,137 +682,4 @@ impl fmt::Debug for Connection { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_https_url() { - let url = - ParsedUrl::parse("https://openrouter.ai/api/v1/chat?x=1").expect("valid https URL"); - assert_eq!(url.scheme, "https"); - assert_eq!(url.host, "openrouter.ai"); - assert_eq!(url.port, 443); - assert_eq!(url.path_and_query, "/api/v1/chat?x=1"); - } - - #[test] - fn rejects_plain_http_non_loopback() { - let error = ParsedUrl::parse("http://example.com/api").expect_err("non-loopback rejected"); - assert!(matches!(error, HttpClientError::PlainHttpNonLoopback)); - } - - #[test] - fn allows_loopback_plain_http() { - let url = ParsedUrl::parse("http://127.0.0.1:8080/ok").expect("loopback allowed"); - assert_eq!(url.port, 8080); - } - - #[test] - fn rejects_https_ip_literal() { - let error = ParsedUrl::parse("https://127.0.0.1/api").expect_err("IP literal rejected"); - assert!(matches!(error, HttpClientError::Url { .. })); - } - - #[test] - fn deadline_stream_read_loop_cannot_outlive_the_absolute_deadline() { - // rustls assembles one TLS record by looping small sock.read() calls; - // a peer trickling bytes resets a socket-level timeout on every byte. - // This drives DeadlineStream exactly that way and proves the absolute - // deadline still fires — the property that makes the TLS path safe. - use std::io::Write as _; - use std::net::TcpListener; - - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); - let addr = listener.local_addr().expect("addr"); - let server = std::thread::Builder::new() - .name("deadline-stream-trickle-server".to_string()) - .spawn(move || { - let (mut stream, _) = listener.accept().expect("accept"); - for _ in 0..10_000 { - std::thread::sleep(Duration::from_millis(40)); - if stream - .write_all(b"x") - .and_then(|()| stream.flush()) - .is_err() - { - break; // client hung up — the intended outcome - } - } - }) - .expect("spawn trickle server"); - - let tcp = TcpStream::connect(addr).expect("connect"); - let deadline = Deadline::new(Duration::from_secs(1)).expect("deadline"); - let mut stream = DeadlineStream::new(tcp, &deadline); - - let started = Instant::now(); - let mut scratch = [0_u8; 1]; - let mut total = 0_usize; - let error = loop { - match stream.read(&mut scratch) { - Ok(0) => { - break io::Error::new(ErrorKind::UnexpectedEof, "peer closed early"); - } - Ok(n) => { - total += n; - assert!( - total < 10_000, - "drained the whole trickle without timing out" - ); - } - Err(error) => break error, - } - }; - let elapsed = started.elapsed(); - assert!( - is_timeout_error(&error), - "expected a timeout error, got {error:?}" - ); - assert!( - elapsed < Duration::from_secs(5), - "read loop ran {elapsed:?} against a 1s deadline" - ); - drop(stream); // break the pipe so the trickler exits - let _ = server.join(); - } - - #[test] - fn deadline_stream_read_fails_closed_once_the_deadline_passed() { - use std::net::TcpListener; - - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); - let addr = listener.local_addr().expect("addr"); - let server = std::thread::Builder::new() - .name("deadline-stream-idle-server".to_string()) - .spawn(move || { - let _accepted = listener.accept(); - std::thread::sleep(Duration::from_millis(200)); - }) - .expect("spawn idle server"); - - let tcp = TcpStream::connect(addr).expect("connect"); - // Already-past deadline: the very first read must fail without blocking. - let deadline = Deadline::new(Duration::from_millis(1)).expect("deadline"); - std::thread::sleep(Duration::from_millis(5)); - let mut stream = DeadlineStream::new(tcp, &deadline); - let mut scratch = [0_u8; 1]; - let error = stream - .read(&mut scratch) - .expect_err("past-deadline read must fail"); - assert!(is_timeout_error(&error), "expected timeout, got {error:?}"); - let _ = server.join(); - } - - #[test] - fn transient_classification_matches_policy() { - let timed_out = HttpClientError::Io { - during: "read", - source: io::Error::new(ErrorKind::TimedOut, "timeout"), - }; - assert!(timed_out.is_transient()); - let malformed = HttpClientError::MalformedResponse { - detail: "bad".to_string(), - }; - assert!(!malformed.is_transient()); - } -} +mod tests; diff --git a/src/surfaces/http/client/tests.rs b/src/surfaces/http/client/tests.rs new file mode 100644 index 0000000..373391d --- /dev/null +++ b/src/surfaces/http/client/tests.rs @@ -0,0 +1,131 @@ +use super::*; + +#[test] +fn parses_https_url() { + let url = ParsedUrl::parse("https://openrouter.ai/api/v1/chat?x=1").expect("valid https URL"); + assert_eq!(url.scheme, "https"); + assert_eq!(url.host, "openrouter.ai"); + assert_eq!(url.port, 443); + assert_eq!(url.path_and_query, "/api/v1/chat?x=1"); +} + +#[test] +fn rejects_plain_http_non_loopback() { + let error = ParsedUrl::parse("http://example.com/api").expect_err("non-loopback rejected"); + assert!(matches!(error, HttpClientError::PlainHttpNonLoopback)); +} + +#[test] +fn allows_loopback_plain_http() { + let url = ParsedUrl::parse("http://127.0.0.1:8080/ok").expect("loopback allowed"); + assert_eq!(url.port, 8080); +} + +#[test] +fn rejects_https_ip_literal() { + let error = ParsedUrl::parse("https://127.0.0.1/api").expect_err("IP literal rejected"); + assert!(matches!(error, HttpClientError::Url { .. })); +} + +#[test] +fn deadline_stream_read_loop_cannot_outlive_the_absolute_deadline() { + // rustls assembles one TLS record by looping small sock.read() calls; + // a peer trickling bytes resets a socket-level timeout on every byte. + // This drives DeadlineStream exactly that way and proves the absolute + // deadline still fires — the property that makes the TLS path safe. + use std::io::Write as _; + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let addr = listener.local_addr().expect("addr"); + let server = std::thread::Builder::new() + .name("deadline-stream-trickle-server".to_string()) + .spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + for _ in 0..10_000 { + std::thread::sleep(Duration::from_millis(40)); + if stream + .write_all(b"x") + .and_then(|()| stream.flush()) + .is_err() + { + break; // client hung up — the intended outcome + } + } + }) + .expect("spawn trickle server"); + + let tcp = TcpStream::connect(addr).expect("connect"); + let deadline = Deadline::new(Duration::from_secs(1)).expect("deadline"); + let mut stream = DeadlineStream::new(tcp, &deadline); + + let started = Instant::now(); + let mut scratch = [0_u8; 1]; + let mut total = 0_usize; + let error = loop { + match stream.read(&mut scratch) { + Ok(0) => { + break io::Error::new(ErrorKind::UnexpectedEof, "peer closed early"); + } + Ok(n) => { + total += n; + assert!( + total < 10_000, + "drained the whole trickle without timing out" + ); + } + Err(error) => break error, + } + }; + let elapsed = started.elapsed(); + assert!( + is_timeout_error(&error), + "expected a timeout error, got {error:?}" + ); + assert!( + elapsed < Duration::from_secs(5), + "read loop ran {elapsed:?} against a 1s deadline" + ); + drop(stream); // break the pipe so the trickler exits + let _ = server.join(); +} + +#[test] +fn deadline_stream_read_fails_closed_once_the_deadline_passed() { + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let addr = listener.local_addr().expect("addr"); + let server = std::thread::Builder::new() + .name("deadline-stream-idle-server".to_string()) + .spawn(move || { + let _accepted = listener.accept(); + std::thread::sleep(Duration::from_millis(200)); + }) + .expect("spawn idle server"); + + let tcp = TcpStream::connect(addr).expect("connect"); + // Already-past deadline: the very first read must fail without blocking. + let deadline = Deadline::new(Duration::from_millis(1)).expect("deadline"); + std::thread::sleep(Duration::from_millis(5)); + let mut stream = DeadlineStream::new(tcp, &deadline); + let mut scratch = [0_u8; 1]; + let error = stream + .read(&mut scratch) + .expect_err("past-deadline read must fail"); + assert!(is_timeout_error(&error), "expected timeout, got {error:?}"); + let _ = server.join(); +} + +#[test] +fn transient_classification_matches_policy() { + let timed_out = HttpClientError::Io { + during: "read", + source: io::Error::new(ErrorKind::TimedOut, "timeout"), + }; + assert!(timed_out.is_transient()); + let malformed = HttpClientError::MalformedResponse { + detail: "bad".to_string(), + }; + assert!(!malformed.is_transient()); +} diff --git a/src/surfaces/http/codec.rs b/src/surfaces/http/codec.rs new file mode 100644 index 0000000..87825ec --- /dev/null +++ b/src/surfaces/http/codec.rs @@ -0,0 +1,102 @@ +//! HTTP transfer-body decoding primitives. + +use std::io::{BufRead, ErrorKind, Read}; + +use super::client::HttpClientError; + +/// Decode an HTTP chunked response body. +/// +/// # Errors +/// +/// Returns [`HttpClientError::MalformedResponse`] for malformed framing, +/// [`HttpClientError::BodyTooLarge`] when the decoded body exceeds `body_cap`, +/// or [`HttpClientError::Io`] when the reader fails. +pub fn decode_chunked( + reader: &mut R, + body_cap: usize, +) -> Result, HttpClientError> { + let mut body = Vec::new(); + loop { + let line = read_line(reader)?; + let size_text = line + .split_once(';') + .map_or(line.as_str(), |(head, _)| head) + .trim(); + if size_text.is_empty() { + return Err(HttpClientError::MalformedResponse { + detail: "empty chunk size".to_string(), + }); + } + let size = usize::from_str_radix(size_text, 16).map_err(|_| { + HttpClientError::MalformedResponse { + detail: "invalid chunk size".to_string(), + } + })?; + if size == 0 { + consume_trailers(reader)?; + return Ok(body); + } + if body.len().saturating_add(size) > body_cap { + return Err(HttpClientError::BodyTooLarge); + } + let start = body.len(); + body.resize(start + size, 0); + read_exact_chunk(reader, &mut body[start..], "chunk data")?; + let mut crlf = [0_u8; 2]; + read_exact_chunk(reader, &mut crlf, "chunk delimiter")?; + if crlf != *b"\r\n" { + return Err(HttpClientError::MalformedResponse { + detail: "chunk data was not followed by CRLF".to_string(), + }); + } + } +} + +fn read_exact_chunk( + reader: &mut R, + buffer: &mut [u8], + during: &'static str, +) -> Result<(), HttpClientError> { + reader.read_exact(buffer).map_err(|source| { + if source.kind() == ErrorKind::UnexpectedEof { + HttpClientError::MalformedResponse { + detail: format!("truncated {during}"), + } + } else { + HttpClientError::Io { during, source } + } + }) +} + +fn consume_trailers(reader: &mut R) -> Result<(), HttpClientError> { + loop { + let line = read_line(reader)?; + if line.is_empty() { + return Ok(()); + } + } +} + +fn read_line(reader: &mut R) -> Result { + let mut bytes = Vec::new(); + let read = reader + .read_until(b'\n', &mut bytes) + .map_err(|source| HttpClientError::Io { + during: "chunk line", + source, + })?; + if read == 0 { + return Err(HttpClientError::MalformedResponse { + detail: "truncated chunked body".to_string(), + }); + } + if !bytes.ends_with(b"\r\n") { + return Err(HttpClientError::MalformedResponse { + detail: "chunk line missing CRLF".to_string(), + }); + } + bytes.truncate(bytes.len() - 2); + String::from_utf8(bytes).map_err(|_| HttpClientError::MalformedResponse { + detail: "chunk line was not UTF-8".to_string(), + }) +} diff --git a/src/surfaces/http/mod.rs b/src/surfaces/http/mod.rs index 52375a8..c2e7c90 100644 --- a/src/surfaces/http/mod.rs +++ b/src/surfaces/http/mod.rs @@ -1,5 +1,11 @@ //! Synchronous HTTP helpers for OpenAI-compatible surfaces. +#[cfg(feature = "openrouter")] +mod codec; +#[cfg(feature = "openrouter")] +mod schedule; +mod types; + /// HTTP/1.1 client. #[cfg(feature = "openrouter")] pub mod client; diff --git a/src/surfaces/http/request.rs b/src/surfaces/http/request.rs index f4b2eb8..15cc989 100644 --- a/src/surfaces/http/request.rs +++ b/src/surfaces/http/request.rs @@ -2,54 +2,11 @@ use std::io::Read; +pub use super::types::{HttpRequest, Method, RequestError}; + const HEAD_CAP: usize = 8 * 1024; const BODY_CAP: usize = 1024 * 1024; -/// Supported HTTP method. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Method { - /// GET. - Get, - /// POST. - Post, -} - -/// Parsed inbound request. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HttpRequest { - /// Request method. - pub method: Method, - /// Path without query. - pub path: String, - /// Query string without `?`. - pub query: Option, - /// Headers in arrival order. - pub headers: Vec<(String, String)>, - /// Exact request body. - pub body: Vec, -} - -/// Parser rejection. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RequestError { - /// HTTP status code. - pub status: u16, - /// JSON error message. - pub message: String, - /// Optional Allow header. - pub allow: Option<&'static str>, -} - -impl RequestError { - fn new(status: u16, message: impl Into) -> Self { - Self { - status, - message: message.into(), - allow: None, - } - } -} - /// Parse one HTTP/1.x request from a blocking stream. /// /// # Errors diff --git a/src/surfaces/http/response.rs b/src/surfaces/http/response.rs index 22656c1..50204cc 100644 --- a/src/surfaces/http/response.rs +++ b/src/surfaces/http/response.rs @@ -2,18 +2,7 @@ use std::io::{self, Write}; -/// Inbound-server HTTP response. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HttpResponse { - /// Numeric status code. - pub status: u16, - /// Extra headers. - pub headers: Vec<(String, String)>, - /// Response body bytes. - pub body: Vec, - /// Whether to add `Connection: close`. - pub close: bool, -} +pub use super::types::HttpResponse; impl HttpResponse { /// Build a response with a byte body. diff --git a/src/surfaces/http/retry.rs b/src/surfaces/http/retry.rs index 88ba810..ccf827f 100644 --- a/src/surfaces/http/retry.rs +++ b/src/surfaces/http/retry.rs @@ -9,21 +9,7 @@ pub const RETRY_BACKOFF: Duration = Duration::from_millis(500); /// Maximum backoff, including `Retry-After` values. pub const MAX_BACKOFF: Duration = Duration::from_secs(30); -/// Compute the retry delay for `attempt`. -/// -/// `attempt` is one-based for retries: the first retry uses `attempt == 1`. -/// Delta-seconds from `Retry-After` override the exponential schedule, and both -/// paths clamp to [`MAX_BACKOFF`]. -#[must_use] -pub fn retry_delay(attempt: u32, retry_after_secs: Option) -> Duration { - let base = if let Some(secs) = retry_after_secs { - Duration::from_secs(secs) - } else { - let shift = attempt.saturating_sub(1).min(20); - RETRY_BACKOFF.saturating_mul(1_u32.checked_shl(shift).unwrap_or(u32::MAX)) - }; - base.min(MAX_BACKOFF) -} +pub use super::schedule::retry_delay; /// Parse a delta-seconds `Retry-After` header from HTTP response headers. #[must_use] diff --git a/src/surfaces/http/schedule.rs b/src/surfaces/http/schedule.rs new file mode 100644 index 0000000..74f8d41 --- /dev/null +++ b/src/surfaces/http/schedule.rs @@ -0,0 +1,21 @@ +//! Retry timing policy. + +use std::time::Duration; + +use super::retry::{MAX_BACKOFF, RETRY_BACKOFF}; + +/// Compute the retry delay for `attempt`. +/// +/// `attempt` is one-based for retries: the first retry uses `attempt == 1`. +/// Delta-seconds from `Retry-After` override the exponential schedule, and both +/// paths clamp to [`MAX_BACKOFF`]. +#[must_use] +pub fn retry_delay(attempt: u32, retry_after_secs: Option) -> Duration { + let base = if let Some(secs) = retry_after_secs { + Duration::from_secs(secs) + } else { + let shift = attempt.saturating_sub(1).min(20); + RETRY_BACKOFF.saturating_mul(1_u32.checked_shl(shift).unwrap_or(u32::MAX)) + }; + base.min(MAX_BACKOFF) +} diff --git a/src/surfaces/http/server.rs b/src/surfaces/http/server.rs index 9e4da21..89092a8 100644 --- a/src/surfaces/http/server.rs +++ b/src/surfaces/http/server.rs @@ -14,22 +14,11 @@ use super::request::{parse, ParseFailure}; use super::response::HttpResponse; use super::routes::{route, RouteState}; +pub use super::types::ServerConfig; + const REQUEST_PERMITS: usize = 64; const SSE_PERMITS: usize = 8; -/// Server configuration. -#[derive(Clone)] -pub struct ServerConfig { - /// Listen address. - pub addr: String, - /// Route state. - pub state: RouteState, - /// Accept-loop idle sleep. - pub idle_sleep: Duration, - /// SSE keep-alive interval. - pub sse_keep_alive: Duration, -} - impl ServerConfig { /// Build a config with default sleeps. #[must_use] @@ -137,7 +126,7 @@ impl AtomicStats { /// Returns [`TexoError::Surface`] when the listener cannot bind or configure. pub fn serve(config: ServerConfig, shutdown: &ShutdownHandle) -> Result { let listener = TcpListener::bind(&config.addr).map_err(surface_error)?; - serve_listener(listener, config, shutdown) + serve_listener(&listener, config, shutdown) } /// Serve an already-bound listener until shutdown. @@ -146,12 +135,8 @@ pub fn serve(config: ServerConfig, shutdown: &ShutdownHandle) -> Result Result { @@ -166,30 +151,26 @@ pub fn serve_listener( match listener.accept() { Ok((stream, _addr)) => { counters.accepted.fetch_add(1, Ordering::AcqRel); - let route_state = Arc::clone(&route_state); - let request_pool = Arc::clone(&request_pool); - let sse_pool = Arc::clone(&sse_pool); - let counters = Arc::clone(&counters); - let shutdown = shutdown.clone(); - let keep_alive = config.sse_keep_alive; - let idle_sleep = config.idle_sleep; + let context = ConnectionContext { + route_state: Arc::clone(&route_state), + request_pool: Arc::clone(&request_pool), + sse_pool: Arc::clone(&sse_pool), + shutdown: shutdown.clone(), + idle_sleep: config.idle_sleep, + keep_alive: config.sse_keep_alive, + counters: Arc::clone(&counters), + }; let worker = std::thread::Builder::new() .name("texo-http-conn".to_string()) .spawn(move || { let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - serve_connection( - stream, - &route_state, - &request_pool, - &sse_pool, - &shutdown, - idle_sleep, - keep_alive, - &counters, - ); + serve_connection(stream, &context); })); if result.is_err() { - counters.worker_panics.fetch_add(1, Ordering::AcqRel); + context + .counters + .worker_panics + .fetch_add(1, Ordering::AcqRel); } }) .map_err(surface_error)?; @@ -212,24 +193,24 @@ pub fn serve_listener( Ok(counters.snapshot()) } -#[expect( - clippy::too_many_arguments, - reason = "connection workers receive independent admission pools, shutdown, timing, and counters" -)] -fn serve_connection( - mut stream: TcpStream, - route_state: &RouteState, - request_pool: &PermitPool, - sse_pool: &PermitPool, - shutdown: &ShutdownHandle, +struct ConnectionContext { + route_state: Arc, + request_pool: Arc, + sse_pool: Arc, + shutdown: ShutdownHandle, idle_sleep: Duration, keep_alive: Duration, - counters: &AtomicStats, -) { + counters: Arc, +} + +fn serve_connection(mut stream: TcpStream, context: &ConnectionContext) { let request = match parse(&mut stream) { Ok(request) => request, Err(ParseFailure::Request(error)) => { - counters.parse_rejections.fetch_add(1, Ordering::AcqRel); + context + .counters + .parse_rejections + .fetch_add(1, Ordering::AcqRel); let mut response = HttpResponse::json_error(error.status, &error.message); if let Some(allow) = error.allow { response @@ -240,35 +221,59 @@ fn serve_connection( return; } Err(ParseFailure::Io(_)) => { - counters.failed_requests.fetch_add(1, Ordering::AcqRel); + context + .counters + .failed_requests + .fetch_add(1, Ordering::AcqRel); return; } }; if request.method == super::request::Method::Get && request.path == "/api/stream" { - let Some(_permit) = sse_pool.acquire(shutdown, idle_sleep) else { + let Some(_permit) = context + .sse_pool + .acquire(&context.shutdown, context.idle_sleep) + else { return; }; let resume_from = super::sse::resume_cursor(&request); - match super::sse::serve(&mut stream, route_state, keep_alive, resume_from, shutdown) { - Ok(()) => counters.served.fetch_add(1, Ordering::AcqRel), - Err(_) => counters.failed_requests.fetch_add(1, Ordering::AcqRel), + match super::sse::serve( + &mut stream, + &context.route_state, + context.keep_alive, + resume_from, + &context.shutdown, + ) { + Ok(()) => context.counters.served.fetch_add(1, Ordering::AcqRel), + Err(_) => context + .counters + .failed_requests + .fetch_add(1, Ordering::AcqRel), }; return; } - let Some(_permit) = request_pool.acquire(shutdown, idle_sleep) else { + let Some(_permit) = context + .request_pool + .acquire(&context.shutdown, context.idle_sleep) + else { return; }; - match route(&request, route_state) { + match route(&request, &context.route_state) { Ok(response) => { if response.status >= 400 { - counters.failed_requests.fetch_add(1, Ordering::AcqRel); + context + .counters + .failed_requests + .fetch_add(1, Ordering::AcqRel); } else { - counters.served.fetch_add(1, Ordering::AcqRel); + context.counters.served.fetch_add(1, Ordering::AcqRel); } let _ = response.write_to(&mut stream); } Err(error) => { - counters.failed_requests.fetch_add(1, Ordering::AcqRel); + context + .counters + .failed_requests + .fetch_add(1, Ordering::AcqRel); let _ = HttpResponse::json_error(500, &error.to_string()).write_to(&mut stream); } } diff --git a/src/surfaces/http/types.rs b/src/surfaces/http/types.rs new file mode 100644 index 0000000..46d1b09 --- /dev/null +++ b/src/surfaces/http/types.rs @@ -0,0 +1,76 @@ +//! Shared inbound HTTP protocol and server configuration shapes. + +use std::time::Duration; + +use super::routes::RouteState; + +/// Supported HTTP method. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Method { + /// GET. + Get, + /// POST. + Post, +} + +/// Parsed inbound request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HttpRequest { + /// Request method. + pub method: Method, + /// Path without query. + pub path: String, + /// Query string without `?`. + pub query: Option, + /// Headers in arrival order. + pub headers: Vec<(String, String)>, + /// Exact request body. + pub body: Vec, +} + +/// Parser rejection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestError { + /// HTTP status code. + pub status: u16, + /// JSON error message. + pub message: String, + /// Optional Allow header. + pub allow: Option<&'static str>, +} + +impl RequestError { + pub(super) fn new(status: u16, message: impl Into) -> Self { + Self { + status, + message: message.into(), + allow: None, + } + } +} + +/// Inbound-server HTTP response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HttpResponse { + /// Numeric status code. + pub status: u16, + /// Extra headers. + pub headers: Vec<(String, String)>, + /// Response body bytes. + pub body: Vec, + /// Whether to add `Connection: close`. + pub close: bool, +} + +/// Server configuration. +#[derive(Clone)] +pub struct ServerConfig { + /// Listen address. + pub addr: String, + /// Route state. + pub state: RouteState, + /// Accept-loop idle sleep. + pub idle_sleep: Duration, + /// SSE keep-alive interval. + pub sse_keep_alive: Duration, +} diff --git a/src/surfaces/openai.rs b/src/surfaces/openai.rs index 162f4d6..8a0fb74 100644 --- a/src/surfaces/openai.rs +++ b/src/surfaces/openai.rs @@ -169,6 +169,15 @@ impl OpenAiCompatClient { /// Returns a typed [`ApiFailure`] for status, transport, deadline, or JSON /// response failures. Provider bodies never enter the error unchanged. pub fn post_json(&self, endpoint: &'static str, body: &Value) -> Result { + if !crate::ops::env::model_calls_allowed() { + return Err(self.failure( + endpoint, + ApiFailureKind::Transport, + None, + 0, + Some("model calls are forbidden during deterministic replay".to_string()), + )); + } let deadline_at = Instant::now() .checked_add(self.request_timeout) .ok_or_else(|| { @@ -429,4 +438,25 @@ mod tests { assert!(!redacted.contains("exact_secret")); assert!(redacted.chars().count() <= MAX_PROVIDER_MESSAGE); } + + #[test] + fn deterministic_replay_cannot_reach_model_transport() { + let client = OpenAiCompatClient::new( + "http://127.0.0.1:9/v1", + "test-key".to_string(), + 0, + Duration::from_secs(1), + ) + .expect("loopback client"); + let error = crate::ops::env::deterministic_projection(|| { + client + .post_json("/chat/completions", &serde_json::json!({})) + .expect_err("replay guard must fail before transport") + }); + assert_eq!(error.kind, ApiFailureKind::Transport); + assert_eq!(error.attempts, 0); + assert!(error + .to_string() + .contains("forbidden during deterministic replay")); + } } diff --git a/src/topology.rs b/src/topology.rs index e2b4eba..d9599d9 100644 --- a/src/topology.rs +++ b/src/topology.rs @@ -4,6 +4,10 @@ use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; +mod model; + +pub use model::TopologyError; + /// Stable workspace-local identity of one physical `BatPak` journal. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(transparent)] @@ -272,62 +276,6 @@ fn valid_env_name(value: &str) -> bool { .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') } -/// Invalid topology declaration. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum TopologyError { - /// No journals were declared. - #[error("workspace topology declares no journals")] - Empty, - /// Invalid stable id. - #[error("invalid journal id: {0}")] - InvalidJournalId(String), - /// Selected or primary journal is absent. - #[error("journal is not declared: {0}")] - MissingJournal(String), - /// Primary journal must own authority. - #[error("primary journal must be canonical: {0}")] - PrimaryNotCanonical(String), - /// Store path is empty. - #[error("journal has an empty store path: {0}")] - EmptyStorePath(String), - /// Canonical journal carried replica-only fields. - #[error("canonical journal carries replica fields: {0}")] - CanonicalHasReplicaFields(String), - /// Replica omitted source. - #[error("replica is missing source_journal: {0}")] - ReplicaMissingSource(String), - /// Replica omitted materialization semantics. - #[error("replica is missing replica_mode: {0}")] - ReplicaMissingMode(String), - /// Remote source fields are incomplete or unsafe. - #[error("replica has invalid remote source fields: {0}")] - InvalidRemoteSource(String), - /// Exact forks require direct source-store access. - #[error("remote replica cannot use exact_fork mode: {0}")] - RemoteExactFork(String), - /// Replica source is not declared. - #[error("replica {replica} references missing source journal {source_journal}")] - MissingSource { - /// Replica id. - replica: String, - /// Missing source id. - source_journal: String, - }, - /// Replica graph contains a cycle. - #[error("replica lineage contains a cycle at {0}")] - ReplicaCycle(String), - /// Two identities alias one configured data directory. - #[error("journals {first} and {second} share store path {path}")] - DuplicateStorePath { - /// First journal id. - first: String, - /// Second journal id. - second: String, - /// Aliased path. - path: String, - }, -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/topology/model.rs b/src/topology/model.rs new file mode 100644 index 0000000..e9ca592 --- /dev/null +++ b/src/topology/model.rs @@ -0,0 +1,57 @@ +//! Topology validation failures. + +/// Invalid topology declaration. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum TopologyError { + /// No journals were declared. + #[error("workspace topology declares no journals")] + Empty, + /// Invalid stable id. + #[error("invalid journal id: {0}")] + InvalidJournalId(String), + /// Selected or primary journal is absent. + #[error("journal is not declared: {0}")] + MissingJournal(String), + /// Primary journal must own authority. + #[error("primary journal must be canonical: {0}")] + PrimaryNotCanonical(String), + /// Store path is empty. + #[error("journal has an empty store path: {0}")] + EmptyStorePath(String), + /// Canonical journal carried replica-only fields. + #[error("canonical journal carries replica fields: {0}")] + CanonicalHasReplicaFields(String), + /// Replica omitted source. + #[error("replica is missing source_journal: {0}")] + ReplicaMissingSource(String), + /// Replica omitted materialization semantics. + #[error("replica is missing replica_mode: {0}")] + ReplicaMissingMode(String), + /// Remote source fields are incomplete or unsafe. + #[error("replica has invalid remote source fields: {0}")] + InvalidRemoteSource(String), + /// Exact forks require direct source-store access. + #[error("remote replica cannot use exact_fork mode: {0}")] + RemoteExactFork(String), + /// Replica source is not declared. + #[error("replica {replica} references missing source journal {source_journal}")] + MissingSource { + /// Replica id. + replica: String, + /// Missing source id. + source_journal: String, + }, + /// Replica graph contains a cycle. + #[error("replica lineage contains a cycle at {0}")] + ReplicaCycle(String), + /// Two identities alias one configured data directory. + #[error("journals {first} and {second} share store path {path}")] + DuplicateStorePath { + /// First journal id. + first: String, + /// Second journal id. + second: String, + /// Aliased path. + path: String, + }, +} diff --git a/tests/agent_catalog.rs b/tests/agent_catalog.rs index 2e80e8d..f233301 100644 --- a/tests/agent_catalog.rs +++ b/tests/agent_catalog.rs @@ -1,9 +1,12 @@ //! Agent catalog discovery, pagination, and workspace-status contracts. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn cli_operation_discovery_uses_the_shared_catalog() -> TestResult { diff --git a/tests/agent_context.rs b/tests/agent_context.rs index 64f7a4d..c239765 100644 --- a/tests/agent_context.rs +++ b/tests/agent_context.rs @@ -1,9 +1,12 @@ //! Agent context integration test. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn agent_context_contains_current_and_stale_claims() -> TestResult { @@ -11,7 +14,7 @@ fn agent_context_contains_current_and_stale_claims() -> TestResult { ingest_courtroom(&mut workspace)?; let context = workspace.invoke( "texo.context.agent", - &json!({"subject": null, "include_stale": true}), + &json!({"subject": null, "include_stale": true, "allow_unsettled": true}), )?; assert_eq!(context["workspace_id"], "demo"); assert!(context["claims"] @@ -37,7 +40,7 @@ fn snapshot_token_keeps_multi_call_reads_on_one_historical_frontier() -> TestRes )?; let first = workspace.invoke( "texo.context.agent", - &json!({"subject": null, "include_stale": true}), + &json!({"subject": null, "include_stale": true, "allow_unsettled": true}), )?; let token = first["snapshot"]["token"] .as_str() @@ -65,12 +68,13 @@ fn snapshot_token_keeps_multi_call_reads_on_one_historical_frontier() -> TestRes &json!({ "subject": null, "include_stale": true, - "snapshot": token + "snapshot": token, + "allow_unsettled": true }), )?; let latest = workspace.invoke( "texo.context.agent", - &json!({"subject": null, "include_stale": true}), + &json!({"subject": null, "include_stale": true, "allow_unsettled": true}), )?; assert_eq!(historical["replayed_through_sequence"], first_frontier); diff --git a/tests/backend_relation_rejudge_durability.rs b/tests/backend_relation_rejudge_durability.rs new file mode 100644 index 0000000..e216700 --- /dev/null +++ b/tests/backend_relation_rejudge_durability.rs @@ -0,0 +1,119 @@ +//! Rejudge append idempotency and reopen-level settlement durability. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +use batpak::event::EventPayload; +use batpak::store::{Freshness, Store, StoreConfig}; +use syncbat::EffectBackend; +use tempfile::TempDir; +use texo::claims::settlement::SettlementCard; +use texo::claims::workspace::WorkspaceCache; +use texo::config::WorkspaceConfig; +use texo::events::coordinate::entity_for_relation_pair; +use texo::events::ids::{relation_pair_id, ClaimId, WorkspaceId}; +use texo::events::payloads::RelationJudgedV1; +use texo::host::{HostFingerprints, HostInterface}; +use texo::journal_store::JournalStore; +use texo::ops::backend::TexoEffectBackend; +use texo::ops::env::{self, OpEnv}; +use texo::relate::settlement::SettledRelation; + +type TestResult = Result<(), Box>; + +fn test_env(root: &TempDir, store: Arc) -> Rc { + Rc::new(OpEnv { + store: JournalStore::writable(store), + workspace_id: "demo".to_string(), + root: root.path().to_path_buf(), + config: WorkspaceConfig::demo(), + cache: RefCell::new(WorkspaceCache::default()), + receipts: RefCell::new(Vec::new()), + observed_at_ms: 1, + host_interface: HostInterface { + schema: "hostbat.interface.v1".to_string(), + version: "test".to_string(), + fingerprints: HostFingerprints { + module_digest: "00".repeat(32), + host_fingerprint: "00".repeat(32), + interface_fingerprint: "00".repeat(32), + }, + operations: Vec::new(), + }, + journal: texo::config::TexoRootConfig::demo() + .resolve_journal(Some("demo"), None) + .expect("test journal") + .1, + }) +} + +fn append_judgment(backend: &mut TexoEffectBackend, payload: &RelationJudgedV1) -> TestResult { + let bytes = batpak::canonical::to_bytes(payload)?; + backend.append_event(RelationJudgedV1::KIND, &bytes)?; + Ok(()) +} + +#[test] +fn changed_same_fingerprint_judgment_survives_reopen_as_later_history() -> TestResult { + let root = TempDir::new()?; + let store_path = root.path().join("store"); + let store = Arc::new(Store::open(StoreConfig::new(&store_path))?); + let op_env = test_env(&root, Arc::clone(&store)); + let guard = env::install(Rc::clone(&op_env)); + let mut backend = TexoEffectBackend; + let workspace = WorkspaceId::try_from("demo")?; + let older = ClaimId::try_from("claim_aaaaaaaaaaaa")?; + let newer = ClaimId::try_from("claim_bbbbbbbbbbbb")?; + let first = RelationJudgedV1 { + workspace_id: workspace.clone(), + older_claim: older.clone(), + newer_claim: newer.clone(), + relation: SettledRelation::Supersedes, + score_ppm: 900_000, + judge_fingerprint: "openrouter:model|relation-v2".to_string(), + cache_key_hex: "first-cache".to_string(), + observed_at_ms: 1, + }; + let later = RelationJudgedV1 { + relation: SettledRelation::Conflicts, + score_ppm: 750_000, + cache_key_hex: "later-cache".to_string(), + observed_at_ms: 2, + ..first.clone() + }; + + append_judgment(&mut backend, &first)?; + append_judgment(&mut backend, &first)?; + append_judgment(&mut backend, &later)?; + append_judgment(&mut backend, &later)?; + + let receipts = op_env.receipts.borrow(); + assert_eq!(receipts.len(), 4); + assert_eq!(receipts[0], receipts[1]); + assert_eq!(receipts[2], receipts[3]); + assert_ne!(receipts[0].event_id_hex, receipts[2].event_id_hex); + drop(receipts); + + let pair_id = relation_pair_id(&workspace, &older, &newer); + let entity = entity_for_relation_pair(pair_id.as_str()); + assert_eq!(store.by_entity(&entity).len(), 2); + + drop(guard); + drop(op_env); + let store = Arc::try_unwrap(store).map_err(|_| "test store still has shared owners")?; + let _closed = store.close()?; + let reopened = Store::open_read_only(StoreConfig::new(store_path))?; + assert_eq!(reopened.by_entity(&entity).len(), 2); + let card = reopened + .project::(&entity, &Freshness::Consistent)? + .ok_or("settlement card missing after reopen")?; + assert_eq!( + card.authoritative.as_ref().map(|row| row.relation), + Some(SettledRelation::Supersedes) + ); + assert_eq!(card.later_judgments.len(), 1); + assert_eq!(card.later_judgments[0].relation, SettledRelation::Conflicts); + assert_eq!(card.later_judgments[0].cache_key_hex, "later-cache"); + Ok(()) +} diff --git a/tests/compile_fail/illegal_claim_reverse.stderr b/tests/compile_fail/illegal_claim_reverse.stderr index 4f691ef..0dc84af 100644 --- a/tests/compile_fail/illegal_claim_reverse.stderr +++ b/tests/compile_fail/illegal_claim_reverse.stderr @@ -2,9 +2,9 @@ error[E0308]: mismatched types --> tests/compile_fail/illegal_claim_reverse.rs:6:5 | 5 | fn reverse(payload: ClaimSupersededV2) -> Transition { - | -------------------------------------------------- expected `batpak::typestate::Transition` because of return type + | -------------------------------------------------- expected `batpak::typestate::Transition` because of return type 6 | supersede_claim(payload) | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Transition`, found `Transition` | - = note: expected struct `batpak::typestate::Transition` - found struct `batpak::typestate::Transition` + = note: expected struct `batpak::typestate::Transition` + found struct `batpak::typestate::Transition` diff --git a/tests/compile_journaled.rs b/tests/compile_journaled.rs index bf0106a..d19f1ea 100644 --- a/tests/compile_journaled.rs +++ b/tests/compile_journaled.rs @@ -1,9 +1,12 @@ //! Compile journaling integration test. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace, OBSERVED_AT_MS}; +use support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; #[test] fn compile_writes_outputs_and_receipt() -> TestResult { @@ -11,7 +14,11 @@ fn compile_writes_outputs_and_receipt() -> TestResult { ingest_courtroom(&mut workspace)?; let output = workspace.invoke( "texo.compile.run", - &json!({"out_dir": "public", "observed_at_ms": OBSERVED_AT_MS + 3}), + &json!({ + "out_dir": "public", + "observed_at_ms": OBSERVED_AT_MS + 3, + "allow_unsettled": true + }), )?; assert!(output.get("receipt").is_some()); for name in [ @@ -22,7 +29,7 @@ fn compile_writes_outputs_and_receipt() -> TestResult { "agent-context.json", "index.html", ] { - assert!(workspace.root().join("public").join(name).exists()); + assert!(workspace.dir.path().join("public").join(name).exists()); } Ok(()) } diff --git a/tests/demo_flow.rs b/tests/demo_flow.rs index de05039..2d90a6b 100644 --- a/tests/demo_flow.rs +++ b/tests/demo_flow.rs @@ -1,9 +1,12 @@ //! Demo flow smoke test. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace, OBSERVED_AT_MS}; +use support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; #[test] fn demo_flow_reaches_verified_compile() -> TestResult { @@ -14,7 +17,11 @@ fn demo_flow_reaches_verified_compile() -> TestResult { assert_eq!(verify["journal_ok"], true); let compile = workspace.invoke( "texo.compile.run", - &json!({"out_dir": "public", "observed_at_ms": OBSERVED_AT_MS + 3}), + &json!({ + "out_dir": "public", + "observed_at_ms": OBSERVED_AT_MS + 3, + "allow_unsettled": true + }), )?; assert!(compile["files"] .as_array() diff --git a/tests/e2e_pipeline.rs b/tests/e2e_pipeline.rs index d1ee6b2..92144dc 100644 --- a/tests/e2e_pipeline.rs +++ b/tests/e2e_pipeline.rs @@ -1,9 +1,12 @@ //! End-to-end pipeline smoke test. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn e2e_pipeline_lists_explains_and_checks_staleness() -> TestResult { diff --git a/tests/eval_helios.rs b/tests/eval_helios.rs deleted file mode 100644 index 9bc0fb9..0000000 --- a/tests/eval_helios.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Ignored live Helios evaluation gate. - -#[test] -#[ignore = "requires TEXO_LLM_API_KEY and live semantic orchestration"] -fn helios_live_eval_is_key_gated() { - let Ok(key) = std::env::var("TEXO_LLM_API_KEY") else { - return; - }; - assert!(!key.trim().is_empty()); -} diff --git a/tests/golden_agent_context.rs b/tests/golden_agent_context.rs index 5e9fabb..49cb95e 100644 --- a/tests/golden_agent_context.rs +++ b/tests/golden_agent_context.rs @@ -1,9 +1,12 @@ //! Golden agent-context snapshot. +#[path = "support/sample.rs"] +mod sample_support; mod support; +use sample_support::ingest_sample_sources; use serde_json::json; -use support::{ingest_sample_sources, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn agent_context_demo() -> TestResult { @@ -11,7 +14,7 @@ fn agent_context_demo() -> TestResult { let _report = ingest_sample_sources(&mut workspace)?; let context = workspace.invoke( "texo.context.agent", - &json!({"subject": null, "include_stale": true}), + &json!({"subject": null, "include_stale": true, "allow_unsettled": true}), )?; insta::assert_json_snapshot!("agent_context_demo", context, { ".claims[].receipt.event_id" => "[event-id]", diff --git a/tests/golden_compile.rs b/tests/golden_compile.rs index a34632c..9693104 100644 --- a/tests/golden_compile.rs +++ b/tests/golden_compile.rs @@ -1,9 +1,12 @@ //! Golden compile snapshot. +#[path = "support/sample.rs"] +mod sample_support; mod support; +use sample_support::ingest_sample_sources; use serde_json::json; -use support::{ingest_sample_sources, TestResult, TestWorkspace, OBSERVED_AT_MS}; +use support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; #[test] fn compile_demo() -> TestResult { @@ -11,10 +14,14 @@ fn compile_demo() -> TestResult { let _report = ingest_sample_sources(&mut workspace)?; let output = workspace.invoke( "texo.compile.run", - &json!({"out_dir": "public", "observed_at_ms": OBSERVED_AT_MS + 3}), + &json!({ + "out_dir": "public", + "observed_at_ms": OBSERVED_AT_MS + 3, + "allow_unsettled": true + }), )?; let onboarding = - std::fs::read_to_string(workspace.root().join("public/onboarding.generated.md"))?; + std::fs::read_to_string(workspace.dir.path().join("public/onboarding.generated.md"))?; insta::assert_json_snapshot!("compile_demo", output, { ".receipt.event_id_hex" => "[event-id]", ".receipt.global_sequence" => "[sequence]" diff --git a/tests/golden_ingest.rs b/tests/golden_ingest.rs index 1b36a44..96bbc65 100644 --- a/tests/golden_ingest.rs +++ b/tests/golden_ingest.rs @@ -1,9 +1,12 @@ //! Golden ingest snapshot. +#[path = "support/sample.rs"] +mod sample_support; mod support; +use sample_support::ingest_sample_sources; use serde_json::json; -use support::{ingest_sample_sources, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn ingest_demo() -> TestResult { diff --git a/tests/golden_staleness.rs b/tests/golden_staleness.rs index 50b52e0..17279b7 100644 --- a/tests/golden_staleness.rs +++ b/tests/golden_staleness.rs @@ -1,9 +1,12 @@ //! Golden staleness snapshot. +#[path = "support/sample.rs"] +mod sample_support; mod support; +use sample_support::ingest_sample_sources; use serde_json::json; -use support::{ingest_sample_sources, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn staleness_stale_onboarding() -> TestResult { diff --git a/tests/helios_e2e.rs b/tests/helios_e2e.rs index deb0b02..a5e98b6 100644 --- a/tests/helios_e2e.rs +++ b/tests/helios_e2e.rs @@ -17,6 +17,8 @@ struct Oracle { #[serde(default)] conflict: Vec, #[serde(default)] + stale_line: Vec, + #[serde(default)] noise: Vec, } @@ -31,6 +33,21 @@ struct ConflictCase { b_contains: String, } +struct TrophySections { + current: String, + stale: String, + conflicts: String, + all: String, +} + +#[derive(Clone, Copy)] +enum TrophyBucket { + Current, + Stale, + Conflicts, + Other, +} + fn repo_path(path: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path) } @@ -82,32 +99,111 @@ fn run_texo(bin: &Path, root: &Path, args: &[&str], cache_root: &Path) -> TestRe Ok(()) } +fn run_relate_to_completion(bin: &Path, root: &Path, cache_root: &Path) -> TestResult { + let mut cursor = 0_u64; + for _ in 0..100 { + let cursor_arg = cursor.to_string(); + let output = Command::new(bin) + .arg("--root") + .arg(root) + .args([ + "relate", + "--json", + "--pair-budget", + "100000", + "--pair-cursor", + cursor_arg.as_str(), + ]) + .env("TEXO_EXTRACT_CACHE", cache_root.join("extract")) + .env("TEXO_RELATE_CACHE", cache_root.join("relate")) + .output()?; + assert!( + matches!(output.status.code(), Some(0 | 2)), + "relate failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let value: serde_json::Value = serde_json::from_slice(&output.stdout)?; + if value.get("outcome").and_then(serde_json::Value::as_str) == Some("complete") { + return Ok(()); + } + let next = value + .get("next_candidate_cursor") + .and_then(serde_json::Value::as_u64) + .ok_or("partial relate output omitted next_candidate_cursor")?; + if next == cursor { + return Err("relate cursor made no progress".into()); + } + cursor = next; + } + Err("relate did not complete within 100 bounded pages".into()) +} + fn contains_ci(haystack: &str, needle: &str) -> bool { haystack .to_ascii_lowercase() .contains(&needle.to_ascii_lowercase()) } +fn trophy_sections(all: String) -> TrophySections { + let mut current = String::new(); + let mut stale = String::new(); + let mut conflicts = String::new(); + let mut selected = TrophyBucket::Other; + for line in all.lines() { + if let Some(title) = line.strip_prefix("## ") { + selected = if title.starts_with("Current") { + TrophyBucket::Current + } else if title.starts_with("Stale") { + TrophyBucket::Stale + } else if title.starts_with("Conflicts") { + TrophyBucket::Conflicts + } else { + TrophyBucket::Other + }; + continue; + } + let section = match selected { + TrophyBucket::Current => Some(&mut current), + TrophyBucket::Stale => Some(&mut stale), + TrophyBucket::Conflicts => Some(&mut conflicts), + TrophyBucket::Other => None, + }; + if let Some(section) = section { + section.push_str(line); + section.push('\n'); + } + } + TrophySections { + current, + stale, + conflicts, + all, + } +} + #[test] #[ignore = "live model e2e; requires TEXO_LLM_API_KEY"] fn helios_live_e2e_reaches_oracle() -> TestResult { - let Ok(key) = std::env::var("TEXO_LLM_API_KEY") else { - return Ok(()); - }; + let key = std::env::var("TEXO_LLM_API_KEY") + .map_err(|_| "explicit Helios live gate requires TEXO_LLM_API_KEY")?; if key.trim().is_empty() { - return Ok(()); + return Err("explicit Helios live gate requires a non-empty TEXO_LLM_API_KEY".into()); } let bin = assert_cmd::cargo::cargo_bin("texo"); let dir = TempDir::new()?; let root = dir.path(); - let cache_root = repo_path(".texo/cache"); + // The explicit live gate owns an empty, per-run cache. A checked-in or + // developer cache must never turn this behavior proof into replay-only + // validation. + let cache_root = root.join(".texo/live-cache"); write_config(root, &bin)?; let docs = repo_path("examples/helios/docs"); let docs_arg = docs.to_string_lossy().into_owned(); run_texo(&bin, root, &["ingest", docs_arg.as_str()], &cache_root)?; - run_texo(&bin, root, &["relate"], &cache_root)?; + run_relate_to_completion(&bin, root, &cache_root)?; run_texo( &bin, root, @@ -115,33 +211,43 @@ fn helios_live_e2e_reaches_oracle() -> TestResult { &cache_root, )?; - let trophy = std::fs::read_to_string(root.join("public/helios/onboarding.generated.md"))?; + let trophy = trophy_sections(std::fs::read_to_string( + root.join("public/helios/onboarding.generated.md"), + )?); let oracle = load_oracle()?; for case in &oracle.current_claim { assert!( - contains_ci(&trophy, &case.text_contains), + contains_ci(&trophy.current, &case.text_contains), "current oracle text missing: {}", case.text_contains ); } for case in &oracle.superseded_claim { assert!( - contains_ci(&trophy, &case.text_contains), + contains_ci(&trophy.stale, &case.text_contains), "superseded oracle text missing: {}", case.text_contains ); } for case in &oracle.conflict { assert!( - contains_ci(&trophy, &case.a_contains) && contains_ci(&trophy, &case.b_contains), + contains_ci(&trophy.conflicts, &case.a_contains) + && contains_ci(&trophy.conflicts, &case.b_contains), "conflict oracle text missing: {} <> {}", case.a_contains, case.b_contains ); } + for case in &oracle.stale_line { + assert!( + contains_ci(&trophy.stale, &case.text_contains), + "stale-line oracle text missing from stale section: {}", + case.text_contains + ); + } for case in &oracle.noise { assert!( - !contains_ci(&trophy, &case.text_contains), + !contains_ci(&trophy.all, &case.text_contains), "noise text appeared in live trophy: {}", case.text_contains ); diff --git a/tests/helios_frozen.rs b/tests/helios_frozen.rs index 9dceb43..fc9ccc7 100644 --- a/tests/helios_frozen.rs +++ b/tests/helios_frozen.rs @@ -1,171 +1,32 @@ -//! Network-free Helios trophy guard. +//! Network-free rendering-format smoke test for the checked-in Helios trophy. +//! +//! Semantic correctness belongs to the key-gated live Helios pipeline. This +//! test deliberately makes no oracle claims about a frozen artifact. use std::path::PathBuf; -use serde::Deserialize; - type TestResult = Result>; -#[derive(Debug, Deserialize)] -struct Oracle { - #[serde(default)] - current_claim: Vec, - #[serde(default)] - superseded_claim: Vec, - #[serde(default)] - conflict: Vec, - #[serde(default)] - stale_line: Vec, - #[serde(default)] - noise: Vec, -} - -#[derive(Debug, Deserialize)] -struct TextCase { - #[serde(default)] - subject: String, - text_contains: String, -} - -#[derive(Debug, Deserialize)] -struct ConflictCase { - a_contains: String, - b_contains: String, -} - -struct Trophy { - current: String, - stale: String, - conflicts: String, - all: String, -} - fn repo_path(path: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path) } -fn load_oracle() -> TestResult { - let path = repo_path("examples/helios/ground_truth.toml"); - let raw = std::fs::read_to_string(&path)?; - Ok(toml::from_str(&raw)?) -} - -fn load_trophy() -> TestResult { - let path = repo_path("examples/helios/onboarding.generated.md"); - let all = std::fs::read_to_string(&path)?; - let mut current = String::new(); - let mut stale = String::new(); - let mut conflicts = String::new(); - let mut bucket = Bucket::Other; - for line in all.lines() { - if let Some(title) = line.strip_prefix("## ") { - bucket = if title.starts_with("Current") { - Bucket::Current - } else if title.starts_with("Stale") { - Bucket::Stale - } else if title.starts_with("Conflicts") { - Bucket::Conflicts - } else { - Bucket::Other - }; - continue; - } - match bucket { - Bucket::Current => push_line(&mut current, line), - Bucket::Stale => push_line(&mut stale, line), - Bucket::Conflicts => push_line(&mut conflicts, line), - Bucket::Other => {} - } - } - Ok(Trophy { - current, - stale, - conflicts, - all, - }) -} - -#[derive(Debug, Clone, Copy)] -enum Bucket { - Current, - Stale, - Conflicts, - Other, -} - -fn push_line(out: &mut String, line: &str) { - out.push_str(line); - out.push('\n'); -} - -fn contains_ci(haystack: &str, needle: &str) -> bool { - haystack - .to_ascii_lowercase() - .contains(&needle.to_ascii_lowercase()) -} - -fn oracle_text_satisfied(section: &str, case: &TextCase) -> bool { - contains_ci(section, &case.text_contains) - || (case.subject == "storage-engine" - && case.text_contains == "uses BatPak" - && contains_ci(section, "BatPak")) -} - #[test] -fn helios_trophy_matches_ground_truth_oracle() -> TestResult { - let oracle = load_oracle()?; - let trophy = load_trophy()?; - - assert!( - !trophy.current.trim().is_empty(), - "Current section missing or empty" - ); +fn helios_trophy_keeps_the_public_rendering_contract() -> TestResult { + let trophy = std::fs::read_to_string(repo_path("examples/helios/onboarding.generated.md"))?; + for heading in ["## Current", "## Stale", "## Conflicts"] { + assert!( + trophy.contains(heading), + "missing rendering section {heading}" + ); + } assert!( - !trophy.stale.trim().is_empty(), - "Stale section missing or empty" + trophy.lines().any(|line| line.contains("source:")), + "rendered claims must expose source provenance" ); assert!( - !trophy.conflicts.trim().is_empty(), - "Conflicts section missing or empty" + trophy.lines().any(|line| line.starts_with("- **claim_")), + "rendered output must contain at least one formatted claim" ); - - for case in &oracle.current_claim { - assert!( - oracle_text_satisfied(&trophy.current, case), - "current claim missing from trophy: {}", - case.text_contains - ); - } - for case in &oracle.superseded_claim { - assert!( - contains_ci(&trophy.stale, &case.text_contains), - "superseded claim missing from trophy stale section: {}", - case.text_contains - ); - } - for case in &oracle.stale_line { - assert!( - contains_ci(&trophy.stale, &case.text_contains), - "stale-line oracle text missing from trophy stale section: {}", - case.text_contains - ); - } - for case in &oracle.conflict { - assert!( - contains_ci(&trophy.conflicts, &case.a_contains) - && contains_ci(&trophy.conflicts, &case.b_contains), - "conflict missing from trophy: {} <> {}", - case.a_contains, - case.b_contains - ); - } - for case in &oracle.noise { - assert!( - !contains_ci(&trophy.all, &case.text_contains), - "noise text appeared in trophy: {}", - case.text_contains - ); - } - Ok(()) } diff --git a/tests/http_server.rs b/tests/http_server.rs index 4f86cd5..0063c3c 100644 --- a/tests/http_server.rs +++ b/tests/http_server.rs @@ -41,7 +41,7 @@ fn start_server(dir: &TempDir, keep_alive: Duration) -> TestResult TestResult { workspace.write("docs/a.md", "Alice owns release approval.\n")?; workspace.write("docs/b.md", "Deploys happen on Friday.\n")?; - std::fs::write(workspace.root().join("docs/bad.md"), [0xff, 0xfe])?; + std::fs::write(workspace.dir.path().join("docs/bad.md"), [0xff, 0xfe])?; Ok(()) } @@ -69,7 +69,7 @@ fn missing_root_fails_and_existing_empty_root_is_stamped_success() -> TestResult .expect_err("missing root must fail"); assert!(missing.to_string().contains("source")); - std::fs::create_dir_all(workspace.root().join("empty"))?; + std::fs::create_dir_all(workspace.dir.path().join("empty"))?; let empty = workspace.invoke( "texo.ingest.run", &json!({ diff --git a/tests/knowledge_index.rs b/tests/knowledge_index.rs index 5d990b2..282243b 100644 --- a/tests/knowledge_index.rs +++ b/tests/knowledge_index.rs @@ -1,5 +1,8 @@ //! Journaled Git snapshot and claim-evidence integration. +#[path = "support/model.rs"] +mod model_support; + use std::path::Path; use std::process::Command; @@ -15,6 +18,8 @@ use texo::events::payloads::{ }; use texo::host::TexoHost; +use model_support::write_model_capable_config; + type TestResult = Result>; fn git(root: &Path, args: &[&str]) -> TestResult { @@ -68,6 +73,7 @@ fn initialized_repository() -> TestResult<(TempDir, TexoHost)> { git(root.path(), &["add", "docs/decision.md", "src/lib.rs"])?; git(root.path(), &["commit", "-qm", "initial"])?; + write_model_capable_config(root.path())?; let mut host = TexoHost::open(root.path(), "demo", 1_700_000_000_000)?; let _ = host.invoke_json("texo.workspace.init", &json!({"workspace_id": "demo"}))?; let _ = host.invoke_json( @@ -105,73 +111,13 @@ fn count_event_kind(host: &TexoHost, kind: EventKind) -> usize { .count() } -#[test] -fn index_journals_snapshot_evidence_links_and_causal_headers_once() -> TestResult { - let (_root, mut host) = initialized_repository()?; - let first = host.invoke_json( - "texo.knowledge.index", - &json!({"observed_at_ms": 1_700_000_000_002_u64}), - )?; - assert_eq!(first["already_indexed"], false); - assert_eq!(first["evidence_recorded"], 1); - assert_eq!(first["claims_linked"], 1); - - let status = host.invoke_json("texo.workspace.status", &json!({}))?; - assert_eq!( - status["snapshot"]["descriptor"]["source_snapshot_id"], - first["snapshot_id"] - ); - assert_ne!(status["coverage"]["analysis_quality"], "unavailable"); - - let region = Region::scope(scope_for_workspace("demo")); - let entries = host.store().query_entries_after(®ion, None, 256); - let kind_entry = |kind: EventKind| { - entries - .iter() - .find(|entry| entry.event_kind() == kind) - .ok_or("event kind") - }; - let snapshot = kind_entry(::KIND)?; - let evidence = kind_entry(::KIND)?; - let link = kind_entry(::KIND)?; - assert_eq!(evidence.correlation_id(), snapshot.event_id().as_u128()); - assert_eq!(evidence.causation_id(), Some(snapshot.event_id().as_u128())); - assert_eq!(link.correlation_id(), snapshot.event_id().as_u128()); - assert_eq!(link.causation_id(), Some(evidence.event_id().as_u128())); - - let count = count_workspace_events(&host); - let second = host.invoke_json( - "texo.knowledge.index", - &json!({"observed_at_ms": 1_700_000_000_003_u64}), - )?; - assert_eq!(second["already_indexed"], true); - assert_eq!(count_workspace_events(&host), count); - Ok(()) -} - -#[test] -fn changed_worktree_never_links_old_claim_as_supporting_evidence() -> TestResult { - let (root, mut host) = initialized_repository()?; - std::fs::write( - root.path().join("docs/decision.md"), - "Decision: deploys happen on Tuesday.\n", - )?; - let output = host.invoke_json( - "texo.knowledge.index", - &json!({"observed_at_ms": 1_700_000_000_002_u64}), - )?; - assert_eq!(output["evidence_recorded"], 0); - assert_eq!(output["claims_linked"], 0); - assert!(output["coverage"]["gaps"] - .as_array() - .is_some_and(|gaps| gaps.iter().any(|gap| { - gap["path"] == "docs/decision.md" && gap["kind"] == "analysis_incomplete" - }))); - Ok(()) +struct ReplacementScenario { + root: TempDir, + host: TexoHost, + superseded_before: usize, } -#[test] -fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> TestResult { +fn concurrent_replacement_scenario() -> TestResult { let root = TempDir::new()?; git(root.path(), &["init", "-q"])?; git(root.path(), &["config", "user.name", "Texo Test"])?; @@ -212,8 +158,16 @@ fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> )?; git(root.path(), &["add", "docs/decision.md"])?; git(root.path(), &["commit", "-qm", "Singapore target"])?; - let before = count_event_kind(&host, ::KIND); - let ingest = host.invoke_json( + let superseded_before = count_event_kind(&host, ::KIND); + Ok(ReplacementScenario { + root, + host, + superseded_before, + }) +} + +fn assert_unknown_replacement_is_held(scenario: &mut ReplacementScenario) -> TestResult { + let ingest = scenario.host.invoke_json( "texo.ingest.run", &json!({"path":"docs/decision.md","dry_run":false,"strict":false,"observed_at_ms":1_700_000_100_003_u64}), )?; @@ -224,11 +178,14 @@ fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> "temporal_unknown" ); assert_eq!( - count_event_kind(&host, ::KIND), - before + count_event_kind(&scenario.host, ::KIND), + scenario.superseded_before ); + Ok(()) +} - let concurrent = host.invoke_json( +fn assert_concurrent_replacement_is_held(scenario: &mut ReplacementScenario) -> TestResult { + let concurrent = scenario.host.invoke_json( "texo.knowledge.index", &json!({"observed_at_ms":1_700_000_100_004_u64}), )?; @@ -239,31 +196,36 @@ fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> "temporal_concurrent" ); assert_eq!( - count_event_kind(&host, ::KIND), - before + count_event_kind(&scenario.host, ::KIND), + scenario.superseded_before ); + Ok(()) +} - git(root.path(), &["checkout", "-q", "left"])?; +fn assert_descendant_replacement_resumes(scenario: &mut ReplacementScenario) -> TestResult { + git(scenario.root.path(), &["checkout", "-q", "left"])?; std::fs::write( - root.path().join("docs/decision.md"), + scenario.root.path().join("docs/decision.md"), "Decision: the deploy target now is Singapore.\n", )?; - git(root.path(), &["add", "docs/decision.md"])?; + git(scenario.root.path(), &["add", "docs/decision.md"])?; git( - root.path(), + scenario.root.path(), &["commit", "-qm", "move target after Frankfurt"], )?; - let descendant = host.invoke_json( + let descendant = scenario.host.invoke_json( "texo.knowledge.index", &json!({"observed_at_ms":1_700_000_100_005_u64}), )?; assert_eq!(descendant["supersessions_applied"], 1); assert_eq!(descendant["supersessions_held"], 0); assert_eq!( - count_event_kind(&host, ::KIND), - before + 1 + count_event_kind(&scenario.host, ::KIND), + scenario.superseded_before + 1 ); - let claims = host.invoke_json("texo.claims.list", &json!({"subject":null,"snapshot":null}))?; + let claims = scenario + .host + .invoke_json("texo.claims.list", &json!({"subject":null,"snapshot":null}))?; assert_eq!( claims["claims"] .as_array() @@ -276,6 +238,79 @@ fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> Ok(()) } +#[test] +fn index_journals_snapshot_evidence_links_and_causal_headers_once() -> TestResult { + let (_root, mut host) = initialized_repository()?; + let first = host.invoke_json( + "texo.knowledge.index", + &json!({"observed_at_ms": 1_700_000_000_002_u64}), + )?; + assert_eq!(first["already_indexed"], false); + assert_eq!(first["evidence_recorded"], 1); + assert_eq!(first["claims_linked"], 1); + + let status = host.invoke_json("texo.workspace.status", &json!({}))?; + assert_eq!( + status["snapshot"]["descriptor"]["source_snapshot_id"], + first["snapshot_id"] + ); + assert_ne!(status["coverage"]["analysis_quality"], "unavailable"); + + let region = Region::scope(scope_for_workspace("demo")); + let entries = host.store().query_entries_after(®ion, None, 256); + let kind_entry = |kind: EventKind| { + entries + .iter() + .find(|entry| entry.event_kind() == kind) + .ok_or("event kind") + }; + let snapshot = kind_entry(::KIND)?; + let evidence = kind_entry(::KIND)?; + let link = kind_entry(::KIND)?; + assert_eq!(evidence.correlation_id(), snapshot.event_id().as_u128()); + assert_eq!(evidence.causation_id(), Some(snapshot.event_id().as_u128())); + assert_eq!(link.correlation_id(), snapshot.event_id().as_u128()); + assert_eq!(link.causation_id(), Some(evidence.event_id().as_u128())); + + let count = count_workspace_events(&host); + let second = host.invoke_json( + "texo.knowledge.index", + &json!({"observed_at_ms": 1_700_000_000_003_u64}), + )?; + assert_eq!(second["already_indexed"], true); + assert_eq!(count_workspace_events(&host), count); + Ok(()) +} + +#[test] +fn changed_worktree_never_links_old_claim_as_supporting_evidence() -> TestResult { + let (root, mut host) = initialized_repository()?; + std::fs::write( + root.path().join("docs/decision.md"), + "Decision: deploys happen on Tuesday.\n", + )?; + let output = host.invoke_json( + "texo.knowledge.index", + &json!({"observed_at_ms": 1_700_000_000_002_u64}), + )?; + assert_eq!(output["evidence_recorded"], 0); + assert_eq!(output["claims_linked"], 0); + assert!(output["coverage"]["gaps"] + .as_array() + .is_some_and(|gaps| gaps.iter().any(|gap| { + gap["path"] == "docs/decision.md" && gap["kind"] == "analysis_incomplete" + }))); + Ok(()) +} + +#[test] +fn explicit_replacement_waits_for_git_authority_then_resumes_without_model() -> TestResult { + let mut scenario = concurrent_replacement_scenario()?; + assert_unknown_replacement_is_held(&mut scenario)?; + assert_concurrent_replacement_is_held(&mut scenario)?; + assert_descendant_replacement_resumes(&mut scenario) +} + #[test] fn explain_and_triangulate_join_exact_evidence_at_one_snapshot() -> TestResult { let (_root, mut host) = initialized_repository()?; @@ -293,6 +328,14 @@ fn explain_and_triangulate_join_exact_evidence_at_one_snapshot() -> TestResult { "texo.knowledge.index", &json!({"observed_at_ms": 1_700_000_000_002_u64}), )?; + let related = host.invoke_json( + "texo.relate.run", + &json!({ + "observed_at_ms": 1_700_000_000_003_u64, + "max_candidate_pairs": 1 + }), + )?; + assert_eq!(related["outcome"], "complete"); let explain = host.invoke_json( "texo.claim.explain", &json!({"claim_id": claim_id, "snapshot": null}), diff --git a/tests/mcp_stdio.rs b/tests/mcp_stdio.rs index ca94d0d..53b825c 100644 --- a/tests/mcp_stdio.rs +++ b/tests/mcp_stdio.rs @@ -1,5 +1,7 @@ //! MCP stdio wire tests. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; use std::io::{BufRead, BufReader, Write}; @@ -8,8 +10,9 @@ use std::process::Command; use std::process::{Child, ChildStdin, ChildStdout, Stdio}; use assert_cmd::prelude::*; +use courtroom_support::ingest_courtroom; use serde_json::{json, Value}; -use support::{ingest_courtroom, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; fn spawn_mcp(root: &Path) -> TestResult<(Child, ChildStdin, BufReader)> { let mut command = Command::cargo_bin("texo")?; @@ -99,17 +102,12 @@ fn assert_triangulation_call( Ok(()) } -#[test] -fn mcp_stdio_full_session() -> TestResult { - let mut workspace = TestWorkspace::new()?; - ingest_courtroom(&mut workspace)?; - let root = workspace.root().to_path_buf(); - let support::TestWorkspace { dir: _dir, host } = workspace; - drop(host); - let (mut child, mut stdin, mut stdout) = spawn_mcp(&root)?; - +fn initialize_mcp_session( + stdin: &mut ChildStdin, + stdout: &mut BufReader, +) -> TestResult { send_json( - &mut stdin, + stdin, &json!({ "jsonrpc": "2.0", "id": 1, @@ -117,24 +115,30 @@ fn mcp_stdio_full_session() -> TestResult { "params": { "protocolVersion": "2025-06-18" } }), )?; - let initialize = read_json(&mut stdout)?; + let initialize = read_json(stdout)?; assert_eq!(initialize["result"]["protocolVersion"], "2025-06-18"); assert_eq!(initialize["result"]["serverInfo"]["name"], "texo"); - send_json( - &mut stdin, + stdin, &json!({"jsonrpc": "2.0", "method": "notifications/initialized"}), )?; + Ok(()) +} +fn list_mcp_tools(stdin: &mut ChildStdin, stdout: &mut BufReader) -> TestResult { send_json( - &mut stdin, + stdin, &json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}), )?; - let tools = read_json(&mut stdout)?; - assert_tool_catalog(&tools)?; + assert_tool_catalog(&read_json(stdout)?) +} +fn search_knowledge( + stdin: &mut ChildStdin, + stdout: &mut BufReader, +) -> TestResult { send_json( - &mut stdin, + stdin, &json!({ "jsonrpc": "2.0", "id": 3, @@ -145,7 +149,7 @@ fn mcp_stdio_full_session() -> TestResult { } }), )?; - let claims = read_json(&mut stdout)?; + let claims = read_json(stdout)?; assert!(claims["result"]["content"][0]["text"] .as_str() .expect("tool response has text content") @@ -169,9 +173,16 @@ fn mcp_stdio_full_session() -> TestResult { claims_json["next_actions"][0]["arguments"]["snapshot_token"], snapshot_token ); + Ok(snapshot_token.to_string()) +} +fn assert_explain_error( + stdin: &mut ChildStdin, + stdout: &mut BufReader, + snapshot_token: &str, +) -> TestResult { send_json( - &mut stdin, + stdin, &json!({ "jsonrpc": "2.0", "id": 4, @@ -185,7 +196,7 @@ fn mcp_stdio_full_session() -> TestResult { } }), )?; - let explain_error = read_json(&mut stdout)?; + let explain_error = read_json(stdout)?; assert_eq!(explain_error["error"]["code"], -32603); assert!(explain_error["error"]["data"]["code"] .as_str() @@ -194,14 +205,31 @@ fn mcp_stdio_full_session() -> TestResult { assert!(explain_error["error"]["data"]["committed"].is_string()); assert!(explain_error["error"]["data"]["retry_safe"].is_boolean()); assert!(explain_error["error"]["data"].get("resume").is_some()); + Ok(()) +} - assert_triangulation_call(&mut stdin, &mut stdout, snapshot_token)?; - +fn assert_parse_error(stdin: &mut ChildStdin, stdout: &mut BufReader) -> TestResult { stdin.write_all(b"{not json\n")?; stdin.flush()?; - let parse_error = read_json(&mut stdout)?; + let parse_error = read_json(stdout)?; assert_eq!(parse_error["error"]["code"], -32700); + Ok(()) +} +#[test] +fn mcp_stdio_full_session() -> TestResult { + let mut workspace = TestWorkspace::new()?; + ingest_courtroom(&mut workspace)?; + let root = workspace.dir.path().to_path_buf(); + let support::TestWorkspace { dir: _dir, host } = workspace; + drop(host); + let (mut child, mut stdin, mut stdout) = spawn_mcp(&root)?; + initialize_mcp_session(&mut stdin, &mut stdout)?; + list_mcp_tools(&mut stdin, &mut stdout)?; + let snapshot_token = search_knowledge(&mut stdin, &mut stdout)?; + assert_explain_error(&mut stdin, &mut stdout, &snapshot_token)?; + assert_triangulation_call(&mut stdin, &mut stdout, &snapshot_token)?; + assert_parse_error(&mut stdin, &mut stdout)?; drop(stdin); let status = child.wait()?; assert!(status.success()); @@ -211,7 +239,7 @@ fn mcp_stdio_full_session() -> TestResult { #[test] fn mcp_input_error_carries_safe_retry_facts() -> TestResult { let workspace = TestWorkspace::new()?; - let root = workspace.root().to_path_buf(); + let root = workspace.dir.path().to_path_buf(); let support::TestWorkspace { dir: _dir, host } = workspace; drop(host); let (mut child, mut stdin, mut stdout) = spawn_mcp(&root)?; diff --git a/tests/ops_kit.rs b/tests/ops_kit.rs index 8c5fb54..0bd2529 100644 --- a/tests/ops_kit.rs +++ b/tests/ops_kit.rs @@ -1,8 +1,4 @@ //! WO-2a operation kit invariants. -#![expect( - missing_docs, - reason = "integration test uses syncbat::operation-generated registration shims" -)] use std::cell::RefCell; use std::rc::Rc; @@ -34,7 +30,6 @@ thread_local! { #[syncbat::operation( descriptor = ROW_VIOLATION, - register = register_row_violation, name = "scripted.row_violation", effect = Inspect, input_schema = "scripted.row_violation.input.v1", @@ -53,7 +48,6 @@ fn row_violation(_input: &[u8], cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerRe #[syncbat::operation( descriptor = UNKNOWN_KIND, - register = register_unknown_kind, name = "scripted.unknown_kind", effect = Persist, input_schema = "scripted.unknown_kind.input.v1", @@ -70,7 +64,6 @@ fn unknown_kind(_input: &[u8], cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerRes #[syncbat::operation( descriptor = CAPABILITY_PROBE, - register = register_capability_probe, name = "scripted.capability_probe", effect = Inspect, input_schema = "scripted.capability_probe.input.v1", @@ -78,11 +71,9 @@ fn unknown_kind(_input: &[u8], cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerRes receipt_kind = "receipt.scripted.capability_probe.v1", requires_capabilities = ["texo.cap.model"] )] -#[expect( - clippy::unnecessary_wraps, - reason = "syncbat handlers must return HandlerResult" -)] -fn capability_probe(_input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerResult { +fn capability_probe(input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerResult { + let _request: serde_json::Value = serde_json::from_slice(input) + .map_err(|error| HandlerError::invalid_input(error.to_string()))?; SCRIPTED_PROBE_RAN.with(|slot| { if let Some(flag) = slot.borrow().as_ref() { flag.store(true, Ordering::SeqCst); @@ -91,6 +82,24 @@ fn capability_probe(_input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> syncbat::Handl Ok(b"{}".to_vec()) } +fn register_row_violation( + builder: &mut syncbat::CoreBuilder, +) -> Result<&mut syncbat::CoreBuilder, syncbat::BuildError> { + builder.register(ROW_VIOLATION, row_violation) +} + +fn register_unknown_kind( + builder: &mut syncbat::CoreBuilder, +) -> Result<&mut syncbat::CoreBuilder, syncbat::BuildError> { + builder.register((*UNKNOWN_KIND).clone(), unknown_kind) +} + +fn register_capability_probe( + builder: &mut syncbat::CoreBuilder, +) -> Result<&mut syncbat::CoreBuilder, syncbat::BuildError> { + builder.register((*CAPABILITY_PROBE).clone(), capability_probe) +} + fn source_payload(source_id: &str) -> SourceObservedV2 { SourceObservedV2 { source_id: source_id.to_string(), @@ -211,9 +220,27 @@ fn capability_gate_denies_before_handler_runs() -> TestResult { #[test] fn catalog_descriptors_validate_and_fingerprints_exist() -> TestResult { - for item in texo::ops::catalog() { + let catalog = texo::ops::catalog(); + for item in &catalog { item.descriptor().validate()?; } + let relate = catalog + .iter() + .find(|item| item.descriptor().name() == "texo.relate.run") + .ok_or("relate operation")?; + let effect_row = relate.descriptor().effect_row(); + assert_eq!( + effect_row + .appends_events() + .iter() + .map(String::as_str) + .collect::>(), + ["evt.e003", "evt.e004", "evt.e009", "evt.e00a", "evt.e012"] + ); + assert!(effect_row + .requires_capabilities() + .iter() + .any(|capability| capability == "texo.cap.model")); let dir = TempDir::new()?; let host = TexoHost::open(dir.path(), "demo", 1)?; let fingerprints = host.fingerprints(); diff --git a/tests/oracle_live.rs b/tests/oracle_live.rs index 8be2d0e..1c8c24d 100644 --- a/tests/oracle_live.rs +++ b/tests/oracle_live.rs @@ -8,7 +8,7 @@ use texo::semantics::pipeline::{ receipt_view, relate_claims, ClaimStatus, ClaimView, RelateThresholds, }; -/// Cluster-first thresholds for gemini-embedding-2, mirroring `texo relate`. +/// Semantic candidate thresholds for gemini-embedding-2, mirroring `texo relate`. const THRESHOLDS: RelateThresholds = RelateThresholds { cluster: 0.65, prefilter: 0.60, @@ -48,9 +48,9 @@ fn claim(seq: u64, text: &str, src: &str) -> (ClaimId, ClaimView) { #[test] #[ignore = "live model call; requires TEXO_LLM_API_KEY"] fn helios_relations_hold_with_real_models() { - if std::env::var("TEXO_LLM_API_KEY").is_err() { - return; - } + let key = + std::env::var("TEXO_LLM_API_KEY").expect("explicit live oracle requires TEXO_LLM_API_KEY"); + assert!(!key.trim().is_empty(), "TEXO_LLM_API_KEY cannot be empty"); let claims = vec![ claim(10, "Deploys happen on Friday.", "02_adr_001.md"), @@ -88,8 +88,12 @@ fn helios_relations_hold_with_real_models() { let relater = OpenRouterRelater::new(None, None).expect("relater"); let out = relate_claims(&claims, &embedder, &relater, THRESHOLDS).expect("relate"); - let edges = &out.supersessions; - let conflicts = &out.conflicts; + let authority = &out + .complete() + .expect("ten claims complete in one page") + .related; + let edges = &authority.supersessions; + let conflicts = &authority.conflicts; let edge = |old_sub: &str, new_sub: &str| { edges diff --git a/tests/relation_campaign_gate.rs b/tests/relation_campaign_gate.rs new file mode 100644 index 0000000..66a6cdc --- /dev/null +++ b/tests/relation_campaign_gate.rs @@ -0,0 +1,341 @@ +//! Durable relation-campaign authority gate integration tests. + +mod support; +#[path = "support/model.rs"] +mod model_support; + +use model_support::write_model_capable_config; +use serde_json::{json, Value}; +use support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; +use tempfile::TempDir; +use texo::events::coordinate::{ + coordinate_for_relation_campaign, coordinate_for_relation_pair, entity_for_relation_campaign, +}; +use texo::events::ids::{relation_pair_id, ClaimId, WorkspaceId}; +use texo::events::payloads::{RelationCampaignCheckpointV1, RelationJudgedV1}; +use texo::host::TexoHost; +use texo::relate::settlement::{CampaignPhase, SettledRelation}; + +fn model_capable_workspace() -> TestResult { + let dir = TempDir::new()?; + write_model_capable_config(dir.path())?; + let mut host = TexoHost::open(dir.path(), "demo", OBSERVED_AT_MS)?; + let _initialized = host.invoke_json("texo.workspace.init", &json!({"workspace_id": "demo"}))?; + Ok(TestWorkspace { dir, host }) +} + +fn event_kinds(workspace: &TestWorkspace) -> Vec { + let scope = texo::events::coordinate::scope_for_workspace(workspace.host.workspace_id()); + workspace + .host + .store() + .by_scope(&scope) + .into_iter() + .map(|entry| entry.event_kind().to_string()) + .collect() +} + +fn domain_event_count(workspace: &TestWorkspace) -> usize { + event_kinds(workspace).len() +} + +fn latest_checkpoint(workspace: &TestWorkspace) -> TestResult { + let store = workspace.host.store(); + let entity = entity_for_relation_campaign(workspace.host.workspace_id()); + let entry = store + .by_entity(&entity) + .into_iter() + .last() + .ok_or("relation campaign checkpoint is absent")?; + let raw = store.read_raw(entry.event_id())?; + Ok(batpak::encoding::from_bytes(&raw.event.payload)?) +} + +fn append_phase( + workspace: &TestWorkspace, + checkpoint: &RelationCampaignCheckpointV1, + phase: CampaignPhase, + observed_at_ms: u64, +) -> TestResult { + let mut checkpoint = checkpoint.clone(); + checkpoint.phase = phase; + checkpoint.observed_at_ms = observed_at_ms; + let coordinate = coordinate_for_relation_campaign(workspace.host.workspace_id())?; + let _receipt = workspace + .host + .store() + .append_typed(&coordinate, &checkpoint)?; + Ok(()) +} + +fn append_judgment(workspace: &TestWorkspace, older: &ClaimId, newer: &ClaimId) -> TestResult { + let workspace_id = WorkspaceId::try_from(workspace.host.workspace_id())?; + let pair_id = relation_pair_id(&workspace_id, older, newer); + let coordinate = coordinate_for_relation_pair(workspace_id.as_str(), pair_id.as_str())?; + let payload = RelationJudgedV1 { + workspace_id, + older_claim: older.clone(), + newer_claim: newer.clone(), + relation: SettledRelation::Unrelated, + score_ppm: 900_000, + judge_fingerprint: "test:relation-campaign-gate".to_string(), + cache_key_hex: "test-cache-key".to_string(), + observed_at_ms: OBSERVED_AT_MS + 3, + }; + let _receipt = workspace.host.store().append_typed(&coordinate, &payload)?; + Ok(()) +} + +fn strict_compile(workspace: &mut TestWorkspace) -> Result { + workspace.invoke( + "texo.compile.run", + &json!({ + "out_dir": "strict-public", + "observed_at_ms": OBSERVED_AT_MS + 10 + }), + ) +} + +fn strict_context(workspace: &mut TestWorkspace) -> Result { + workspace.invoke( + "texo.context.agent", + &json!({ + "subject": null, + "include_stale": true + }), + ) +} + +fn workspace_status(workspace: &mut TestWorkspace) -> Result { + workspace.invoke("texo.workspace.status", &json!({"snapshot": null})) +} + +fn assert_settlement_refusal(error: &texo::error::TexoError) { + assert_eq!(error.code(), "op.runtime"); + assert!( + error.to_string().contains("strict settlement refused"), + "unexpected settlement refusal: {error}" + ); +} + +fn claim_rows(workspace: &mut TestWorkspace) -> TestResult> { + let listed = workspace.invoke("texo.claims.list", &json!({"subject": null}))?; + Ok(listed["claims"].as_array().cloned().unwrap_or_default()) +} + +#[test] +fn relate_requires_model_capability_before_handler_execution() -> TestResult { + let mut workspace = TestWorkspace::new()?; + let before = domain_event_count(&workspace); + let error = workspace + .invoke( + "texo.relate.run", + &json!({ + "observed_at_ms": OBSERVED_AT_MS + 1, + "max_candidate_pairs": 1 + }), + ) + .expect_err("relate without model capability must be denied"); + assert_eq!(error.code(), "op.denied"); + assert!(error.to_string().contains("texo.cap.model")); + assert_eq!(domain_event_count(&workspace), before); + Ok(()) +} + +#[test] +fn strict_outputs_require_latest_exact_complete_campaign() -> TestResult { + let mut workspace = model_capable_workspace()?; + let initialized_events = domain_event_count(&workspace); + let initialized_kinds = event_kinds(&workspace); + + let absent = strict_compile(&mut workspace).expect_err("absent checkpoint must block compile"); + assert_settlement_refusal(&absent); + assert_eq!( + domain_event_count(&workspace), + initialized_events, + "event kinds changed from {initialized_kinds:?} to {:?}", + event_kinds(&workspace) + ); + assert!(!workspace.dir.path().join("strict-public").exists()); + assert_eq!( + workspace_status(&mut workspace)?["settlement_complete"], + false + ); + + let relate = workspace.invoke( + "texo.relate.run", + &json!({ + "observed_at_ms": OBSERVED_AT_MS + 1, + "max_candidate_pairs": 1 + }), + )?; + assert_eq!(relate["outcome"], "complete"); + let complete = latest_checkpoint(&workspace)?; + + append_phase( + &workspace, + &complete, + CampaignPhase::Partial { + next_candidate_cursor: 7, + }, + OBSERVED_AT_MS + 2, + )?; + let partial_events = domain_event_count(&workspace); + let context_error = strict_context(&mut workspace).expect_err("partial must block context"); + assert_settlement_refusal(&context_error); + let compile_error = strict_compile(&mut workspace).expect_err("partial must block compile"); + assert_settlement_refusal(&compile_error); + assert_eq!(domain_event_count(&workspace), partial_events); + assert!(!workspace.dir.path().join("strict-public").exists()); + assert_eq!( + workspace_status(&mut workspace)?["settlement_complete"], + false + ); + + append_phase( + &workspace, + &complete, + CampaignPhase::Complete, + OBSERVED_AT_MS + 3, + )?; + assert_eq!( + workspace_status(&mut workspace)?["settlement_complete"], + true + ); + let _context = strict_context(&mut workspace)?; + let compile = strict_compile(&mut workspace)?; + assert!(compile["files"] + .as_array() + .is_some_and(|files| !files.is_empty())); + assert!(workspace + .dir + .path() + .join("strict-public/onboarding.generated.md") + .exists()); + + let before_noop = domain_event_count(&workspace); + let noop = workspace.invoke( + "texo.relate.run", + &json!({ + "observed_at_ms": OBSERVED_AT_MS + 20, + "max_candidate_pairs": 1 + }), + )?; + assert_eq!(noop["outcome"], "complete"); + assert_eq!(domain_event_count(&workspace), before_noop); + Ok(()) +} + +#[test] +fn a_new_claim_invalidates_completion_after_it_becomes_noncurrent() -> TestResult { + let mut workspace = model_capable_workspace()?; + workspace.write("docs/current.md", "Deployments happen on Tuesday.\n")?; + let _ingest = workspace.invoke( + "texo.ingest.run", + &json!({ + "path": "docs/current.md", + "dry_run": false, + "observed_at_ms": OBSERVED_AT_MS + 1 + }), + )?; + let original_claim = claim_rows(&mut workspace)? + .into_iter() + .next() + .and_then(|claim| claim["claim_id"].as_str().map(str::to_string)) + .ok_or("original claim is absent")?; + let relate = workspace.invoke( + "texo.relate.run", + &json!({ + "observed_at_ms": OBSERVED_AT_MS + 2, + "max_candidate_pairs": 1 + }), + )?; + assert_eq!(relate["outcome"], "complete"); + let _context = strict_context(&mut workspace)?; + + workspace.write("docs/retired.md", "Deployments used to happen on Friday.\n")?; + let _ingest = workspace.invoke( + "texo.ingest.run", + &json!({ + "path": "docs/retired.md", + "dry_run": false, + "observed_at_ms": OBSERVED_AT_MS + 3 + }), + )?; + let added_claim = claim_rows(&mut workspace)? + .into_iter() + .filter_map(|claim| claim["claim_id"].as_str().map(str::to_string)) + .find(|claim_id| claim_id != &original_claim) + .ok_or("added claim is absent")?; + let _supersede = workspace.invoke( + "texo.claim.supersede", + &json!({ + "old": added_claim.as_str(), + "new": original_claim.as_str(), + "reason": "historical claim retired", + "decided_by": "human", + "observed_at_ms": OBSERVED_AT_MS + 4 + }), + )?; + let rows = claim_rows(&mut workspace)?; + assert!(rows + .iter() + .any(|claim| { claim["claim_id"] == added_claim && claim["status"] == "superseded" })); + + let error = strict_context(&mut workspace) + .expect_err("new ineligible claim must invalidate prior completion"); + assert_settlement_refusal(&error); + Ok(()) +} + +#[test] +fn rejudge_rejects_a_pair_that_left_the_current_claim_set() -> TestResult { + let mut workspace = model_capable_workspace()?; + workspace.write("docs/older.md", "Decision: deployments happen on Friday.\n")?; + workspace.write( + "docs/newer.md", + "Decision: deployments now happen on Tuesday.\n", + )?; + for (offset, path) in ["docs/older.md", "docs/newer.md"].into_iter().enumerate() { + let _ingest = workspace.invoke( + "texo.ingest.run", + &json!({ + "path": path, + "dry_run": false, + "observed_at_ms": OBSERVED_AT_MS + offset as u64 + }), + )?; + } + let claims = claim_rows(&mut workspace)?; + let [older_row, newer_row] = claims.as_slice() else { + return Err(format!("expected two claims, got {}", claims.len()).into()); + }; + let older = ClaimId::try_from(older_row["claim_id"].as_str().ok_or("older claim id")?)?; + let newer = ClaimId::try_from(newer_row["claim_id"].as_str().ok_or("newer claim id")?)?; + append_judgment(&workspace, &older, &newer)?; + let _supersede = workspace.invoke( + "texo.claim.supersede", + &json!({ + "old": older, + "new": newer, + "reason": "new schedule is authoritative", + "decided_by": "human", + "observed_at_ms": OBSERVED_AT_MS + 4 + }), + )?; + let before = domain_event_count(&workspace); + let error = workspace + .invoke( + "texo.relate.run", + &json!({ + "observed_at_ms": OBSERVED_AT_MS + 5, + "max_candidate_pairs": 1, + "rejudge_pair": [older, newer] + }), + ) + .expect_err("noncurrent rejudge pair must be rejected"); + assert_eq!(error.code(), "op.input"); + assert!(error.to_string().contains("no longer present")); + assert_eq!(domain_event_count(&workspace), before); + Ok(()) +} diff --git a/tests/remote_replication.rs b/tests/remote_replication.rs index 618efcd..3377f5c 100644 --- a/tests/remote_replication.rs +++ b/tests/remote_replication.rs @@ -16,6 +16,7 @@ use texo::topology::JournalEntry; type TestResult = Result>; type ServerThread = JoinHandle>; +const REPLICA_TOKEN: &str = "correct horse battery staple"; fn write_topology(root: &std::path::Path, endpoint: SocketAddr) -> TestResult { let journals = BTreeMap::from([ @@ -144,83 +145,58 @@ fn replica_event_count(root: &std::path::Path) -> TestResult { Ok(store.stats().event_count) } -#[test] -fn remote_replica_authenticates_binds_resumes_and_deduplicates() -> TestResult { - let root = tempfile::tempdir()?; - let listener = TcpListener::bind("127.0.0.1:0")?; - let endpoint = listener.local_addr()?; - write_topology(root.path(), endpoint)?; - ingest( - root.path(), - "docs/one.md", - "Decision: deploys happen on Friday.\n", - 1, - )?; - let (shutdown, handle) = start_server(root.path(), listener)?; - - let denied = replica_command(root.path(), "wrong", "bootstrap", "remote")?; +fn assert_remote_rejections(root: &std::path::Path) -> TestResult { + let denied = replica_command(root, "wrong", "bootstrap", "remote")?; assert!(!denied.status.success()); - assert!(!root.path().join(".texo/replicas/remote").exists()); - let wrong_source = replica_command( - root.path(), - "correct horse battery staple", - "bootstrap", - "wrong-source", - )?; + assert!(!root.join(".texo/replicas/remote").exists()); + let wrong_source = replica_command(root, REPLICA_TOKEN, "bootstrap", "wrong-source")?; assert!(!wrong_source.status.success()); - assert!(!root.path().join(".texo/replicas/wrong-source").exists()); + assert!(!root.join(".texo/replicas/wrong-source").exists()); + Ok(()) +} - let initial = replica_command( - root.path(), - "correct horse battery staple", - "bootstrap", - "remote", - )?; +fn bootstrap_remote_replica( + root: &std::path::Path, + shutdown: &ShutdownHandle, + handle: ServerThread, +) -> TestResult> { + let initial = replica_command(root, REPLICA_TOKEN, "bootstrap", "remote")?; assert!( initial.status.success(), "{}", String::from_utf8_lossy(&initial.stderr) ); - stop_server(&shutdown, handle)?; - assert_eq!( - claims(root.path(), "canonical")?, - claims(root.path(), "remote")? - ); + stop_server(shutdown, handle)?; + assert_eq!(claims(root, "canonical")?, claims(root, "remote")?); + Ok(std::fs::read( + root.join(".texo/replication/demo/remote/cursor.msgpack"), + )?) +} - let stale_cursor = std::fs::read( - root.path() - .join(".texo/replication/demo/remote/cursor.msgpack"), - )?; +fn assert_follow_resumes_and_deduplicates( + root: &std::path::Path, + endpoint: SocketAddr, + stale_cursor: &[u8], +) -> TestResult { ingest( - root.path(), + root, "docs/two.md", "Decision: deploys moved to Tuesday.\n", 2, )?; let listener = TcpListener::bind(endpoint)?; - let (shutdown, handle) = start_server(root.path(), listener)?; - let advanced = replica_command( - root.path(), - "correct horse battery staple", - "follow", - "remote", - )?; + let (shutdown, handle) = start_server(root, listener)?; + let advanced = replica_command(root, REPLICA_TOKEN, "follow", "remote")?; assert!( advanced.status.success(), "{}", String::from_utf8_lossy(&advanced.stderr) ); std::fs::write( - root.path() - .join(".texo/replication/demo/remote/cursor.msgpack"), + root.join(".texo/replication/demo/remote/cursor.msgpack"), stale_cursor, )?; - let replay = replica_command( - root.path(), - "correct horse battery staple", - "follow", - "remote", - )?; + let replay = replica_command(root, REPLICA_TOKEN, "follow", "remote")?; assert!( replay.status.success(), "{}", @@ -231,19 +207,29 @@ fn remote_replica_authenticates_binds_resumes_and_deduplicates() -> TestResult { assert!(replay["deduplicated"] .as_u64() .is_some_and(|count| count > 0)); - let before_noop = replica_event_count(root.path())?; - let no_op = replica_command( - root.path(), - "correct horse battery staple", - "follow", - "remote", - )?; + let before_noop = replica_event_count(root)?; + let no_op = replica_command(root, REPLICA_TOKEN, "follow", "remote")?; assert!(no_op.status.success()); - assert_eq!(replica_event_count(root.path())?, before_noop); + assert_eq!(replica_event_count(root)?, before_noop); stop_server(&shutdown, handle)?; - assert_eq!( - claims(root.path(), "canonical")?, - claims(root.path(), "remote")? - ); + assert_eq!(claims(root, "canonical")?, claims(root, "remote")?); Ok(()) } + +#[test] +fn remote_replica_authenticates_binds_resumes_and_deduplicates() -> TestResult { + let root = tempfile::tempdir()?; + let listener = TcpListener::bind("127.0.0.1:0")?; + let endpoint = listener.local_addr()?; + write_topology(root.path(), endpoint)?; + ingest( + root.path(), + "docs/one.md", + "Decision: deploys happen on Friday.\n", + 1, + )?; + let (shutdown, handle) = start_server(root.path(), listener)?; + assert_remote_rejections(root.path())?; + let stale_cursor = bootstrap_remote_replica(root.path(), &shutdown, handle)?; + assert_follow_resumes_and_deduplicates(root.path(), endpoint, &stale_cursor) +} diff --git a/tests/session_lanes.rs b/tests/session_lanes.rs index f329913..1212560 100644 --- a/tests/session_lanes.rs +++ b/tests/session_lanes.rs @@ -75,7 +75,7 @@ fn turns_survive_crash_before_session_end() -> TestResult { assert_eq!(ended["supersessions_held"], 0); assert_eq!(ended["held_supersessions"], json!([])); let expected_status = - if texo::host::grants_model_capability(std::env::var("TEXO_LLM_API_KEY").ok()) { + if texo::host::grants_model_capability(std::env::var("TEXO_LLM_API_KEY").ok().as_deref()) { "ran" } else { "skipped" diff --git a/tests/spike_family.rs b/tests/spike_family.rs index 1c06ec8..cbcc10e 100644 --- a/tests/spike_family.rs +++ b/tests/spike_family.rs @@ -47,7 +47,6 @@ impl SpikeState { output_schema = "schema.spike.echo.output.v1", receipt_kind = "receipt.spike.echo.v1" )] -#[allow(clippy::unnecessary_wraps)] fn spike_echo(input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerResult { let prefix = SPIKE_ENV.with(|slot| { slot.borrow() @@ -56,7 +55,9 @@ fn spike_echo(input: &[u8], _cx: &mut syncbat::Ctx<'_>) -> syncbat::HandlerResul }); let mut output = prefix.into_bytes(); output.extend_from_slice(b":"); - output.extend_from_slice(input); + let text = std::str::from_utf8(input) + .map_err(|error| syncbat::HandlerError::invalid_input(error.to_string()))?; + output.extend_from_slice(text.as_bytes()); Ok(output) } diff --git a/tests/staleness_courtroom.rs b/tests/staleness_courtroom.rs index c0b19c1..cc2f255 100644 --- a/tests/staleness_courtroom.rs +++ b/tests/staleness_courtroom.rs @@ -1,9 +1,12 @@ //! Staleness courtroom integration test. +#[path = "support/courtroom.rs"] +mod courtroom_support; mod support; +use courtroom_support::ingest_courtroom; use serde_json::json; -use support::{ingest_courtroom, TestResult, TestWorkspace}; +use support::{TestResult, TestWorkspace}; #[test] fn stale_source_line_reports_supersession() -> TestResult { diff --git a/tests/support/courtroom.rs b/tests/support/courtroom.rs new file mode 100644 index 0000000..0fd4d7c --- /dev/null +++ b/tests/support/courtroom.rs @@ -0,0 +1,20 @@ +//! Deploy-change fixture used only by integration tests that request it. + +use serde_json::json; + +use crate::support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; + +/// Populate and ingest the deterministic deploy-change fixture. +pub fn ingest_courtroom(workspace: &mut TestWorkspace) -> TestResult { + workspace.write("docs/friday.md", "Deploys happen on Friday.\n")?; + workspace.write("docs/tuesday.md", "Decision: deploys moved to Tuesday.\n")?; + let _first = workspace.invoke( + "texo.ingest.run", + &json!({"path": "docs/friday.md", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS + 1}), + )?; + let _second = workspace.invoke( + "texo.ingest.run", + &json!({"path": "docs/tuesday.md", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS + 2}), + )?; + Ok(()) +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 8004063..30337d4 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -1,7 +1,5 @@ //! Shared integration-test helpers. -use std::path::Path; - use serde_json::{json, Value}; use tempfile::TempDir; use texo::host::TexoHost; @@ -55,65 +53,4 @@ impl TestWorkspace { pub fn invoke(&mut self, op: &str, input: &Value) -> Result { self.host.invoke_json(op, input) } - - /// Return the temp root path. - #[must_use] - #[allow(dead_code)] - pub fn root(&self) -> &Path { - self.dir.path() - } -} - -/// Copy the repo's bundled `sample_sources/` demo corpus into the workspace -/// root for hermetic tests (the corpus the pre-v2 goldens photographed). -/// -/// # Errors -/// -/// Returns an error when directory creation or a file copy fails. -#[allow(dead_code)] -pub fn copy_sample_sources(workspace: &TestWorkspace) -> TestResult { - let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("sample_sources"); - let dest = workspace.root().join("sample_sources"); - std::fs::create_dir_all(&dest)?; - for entry in std::fs::read_dir(src)? { - let entry = entry?; - std::fs::copy(entry.path(), dest.join(entry.file_name()))?; - } - Ok(()) -} - -/// Ingest the copied demo corpus at the pinned fixture timestamp and return -/// the ingest report. -/// -/// # Errors -/// -/// Returns an error when the ingest op fails. -#[allow(dead_code)] -pub fn ingest_sample_sources(workspace: &mut TestWorkspace) -> TestResult { - copy_sample_sources(workspace)?; - let report = workspace.invoke( - "texo.ingest.run", - &json!({"path": "sample_sources", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS}), - )?; - Ok(report) -} - -/// Populate the courtroom deploy-change fixture. -/// -/// # Errors -/// -/// Returns an error when writing or ingesting fixture files fails. -#[allow(dead_code)] -pub fn ingest_courtroom(workspace: &mut TestWorkspace) -> TestResult { - workspace.write("docs/friday.md", "Deploys happen on Friday.\n")?; - workspace.write("docs/tuesday.md", "Decision: deploys moved to Tuesday.\n")?; - let _first = workspace.invoke( - "texo.ingest.run", - &json!({"path": "docs/friday.md", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS + 1}), - )?; - let _second = workspace.invoke( - "texo.ingest.run", - &json!({"path": "docs/tuesday.md", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS + 2}), - )?; - Ok(()) } diff --git a/tests/support/model.rs b/tests/support/model.rs new file mode 100644 index 0000000..e583a73 --- /dev/null +++ b/tests/support/model.rs @@ -0,0 +1,23 @@ +//! Model-capability setup for integration tests that must not call transport. + +use std::path::Path; + +use texo::config::TexoRootConfig; +use texo::gateway::{GatewayConfig, ProviderProfile, DEFAULT_PROVIDER_ID}; + +/// Configure a model-capable test host without storing or mutating a secret. +pub fn write_model_capable_config(root: &Path) -> Result<(), Box> { + let mut gateway = GatewayConfig::default(); + gateway.providers.insert( + DEFAULT_PROVIDER_ID.to_string(), + ProviderProfile { + base_url: "https://model-transport-must-not-run.invalid/v1".to_string(), + api_key_env: "PATH".to_string(), + ..ProviderProfile::default() + }, + ); + let mut config = TexoRootConfig::demo(); + config.gateway = Some(gateway); + config.save(&root.join(".texo/config.toml"))?; + Ok(()) +} diff --git a/tests/support/sample.rs b/tests/support/sample.rs new file mode 100644 index 0000000..16a06f3 --- /dev/null +++ b/tests/support/sample.rs @@ -0,0 +1,27 @@ +//! Demo-corpus fixture used only by integration tests that request it. + +use std::path::Path; + +use serde_json::{json, Value}; + +use crate::support::{TestResult, TestWorkspace, OBSERVED_AT_MS}; + +fn copy_sample_sources(workspace: &TestWorkspace) -> TestResult { + let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("sample_sources"); + for entry in std::fs::read_dir(source)? { + let entry = entry?; + let relative = format!("sample_sources/{}", entry.file_name().to_string_lossy()); + let text = std::fs::read_to_string(entry.path())?; + workspace.write(&relative, &text)?; + } + Ok(()) +} + +/// Copy and ingest the bundled deterministic demo corpus. +pub fn ingest_sample_sources(workspace: &mut TestWorkspace) -> TestResult { + copy_sample_sources(workspace)?; + Ok(workspace.invoke( + "texo.ingest.run", + &json!({"path": "sample_sources", "dry_run": false, "observed_at_ms": OBSERVED_AT_MS}), + )?) +} diff --git a/tests/topology_scaleout.rs b/tests/topology_scaleout.rs index c77ed4a..36e57ef 100644 --- a/tests/topology_scaleout.rs +++ b/tests/topology_scaleout.rs @@ -94,7 +94,7 @@ fn snapshot_tokens_refuse_a_different_physical_journal() -> TestResult { let _init = canonical.invoke_json("texo.workspace.init", &json!({"workspace_id": "demo"}))?; let context = canonical.invoke_json( "texo.context.agent", - &json!({"subject": null, "include_stale": false}), + &json!({"subject": null, "include_stale": false, "allow_unsettled": true}), )?; let token = context["snapshot"]["token"] .as_str()